go-yaml/lexer/lexer_test.go

package lexer_test

import (
	"sort"
	"testing"

	"github.com/goccy/go-yaml/lexer"
	"github.com/goccy/go-yaml/token"
)

func TestTokenize(t *testing.T) {
	tests := []struct {
		YAML   string
		Tokens token.Tokens
	}{
		{
			YAML: `null
  `,
			Tokens: token.Tokens{
				{
					Type:          token.NullType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "null",
					Origin:        "null\n  ",
				},
			},
		},
		{
			YAML: `0_`,
			Tokens: token.Tokens{
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "0_",
					Origin:        "0_",
				},
			},
		},
		{
			YAML: `"hello\tworld"`,
			Tokens: token.Tokens{
				{
					Type:          token.DoubleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "hello\tworld",
					Origin:        `"hello\tworld"`,
				},
			},
		},
		{
			YAML: `0x_1A_2B_3C`,
			Tokens: token.Tokens{
				{
					Type:          token.HexIntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "0x_1A_2B_3C",
					Origin:        "0x_1A_2B_3C",
				},
			},
		},
		{
			YAML: `+0b1010`,
			Tokens: token.Tokens{
				{
					Type:          token.BinaryIntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "+0b1010",
					Origin:        "+0b1010",
				},
			},
		},
		{
			YAML: `0100`,
			Tokens: token.Tokens{
				{
					Type:          token.OctetIntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "0100",
					Origin:        "0100",
				},
			},
		},
		{
			YAML: `0o10`,
			Tokens: token.Tokens{
				{
					Type:          token.OctetIntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "0o10",
					Origin:        "0o10",
				},
			},
		},
		{
			YAML: `0.123e+123`,
			Tokens: token.Tokens{
				{
					Type:          token.FloatType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "0.123e+123",
					Origin:        "0.123e+123",
				},
			},
		},
		{
			YAML: `{}
  `,
			Tokens: token.Tokens{
				{
					Type:          token.MappingStartType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.FlowCollectionIndicator,
					Value:         "{",
					Origin:        "{",
				},
				{
					Type:          token.MappingEndType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.FlowCollectionIndicator,
					Value:         "}",
					Origin:        "}",
				},
			},
		},
		{
			YAML: `v: hi`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "hi",
					Origin:        " hi",
				},
			},
		},
		{
			YAML: `v:	a`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					// nolint: gci
					Origin: "	a",
				},
			},
		},
		{
			YAML: `v: "true"`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.DoubleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "true",
					Origin:        " \"true\"",
				},
			},
		},
		{
			YAML: `v: "false"`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.DoubleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "false",
					Origin:        " \"false\"",
				},
			},
		},
		{
			YAML: `v: true`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.BoolType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "true",
					Origin:        " true",
				},
			},
		},
		{
			YAML: `v: false`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.BoolType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "false",
					Origin:        " false",
				},
			},
		},
		{
			YAML: `v: 10`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "10",
					Origin:        " 10",
				},
			},
		},
		{
			YAML: `v: -10`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "-10",
					Origin:        " -10",
				},
			},
		},
		{
			YAML: `v: 42`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "42",
					Origin:        " 42",
				},
			},
		},
		{
			YAML: `v: 4294967296`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "4294967296",
					Origin:        " 4294967296",
				},
			},
		},
		{
			YAML: `v: "10"`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.DoubleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "10",
					Origin:        " \"10\"",
				},
			},
		},
		{
			YAML: `v: 0.1`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.FloatType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "0.1",
					Origin:        " 0.1",
				},
			},
		},
		{
			YAML: `v: 0.99`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.FloatType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "0.99",
					Origin:        " 0.99",
				},
			},
		},
		{
			YAML: `v: -0.1`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.FloatType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "-0.1",
					Origin:        " -0.1",
				},
			},
		},
		{
			YAML: `v: .inf`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.InfinityType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         ".inf",
					Origin:        " .inf",
				},
			},
		},
		{
			YAML: `v: -.inf`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.InfinityType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "-.inf",
					Origin:        " -.inf",
				},
			},
		},
		{
			YAML: `v: .nan`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.NanType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         ".nan",
					Origin:        " .nan",
				},
			},
		},
		{
			YAML: `
a:
  "bbb  \
      ccc

      ddd eee\n\
  \ \ fff ggg\nhhh iii\n
  jjj kkk
  "
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "\na",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.DoubleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "bbb  ccc\nddd eee\n  fff ggg\nhhh iii\n jjj kkk ",
					Origin:        "\n  \"bbb  \\\n      ccc\n\n      ddd eee\\n\\\n  \\ \\ fff ggg\\nhhh iii\\n\n  jjj kkk\n  \"",
				},
			},
		},
		{
			YAML: `v: null`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.NullType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "null",
					Origin:        " null",
				},
			},
		},
		{
			YAML: `v: ""`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "v",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.DoubleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "",
					Origin:        " \"\"",
				},
			},
		},
		{
			YAML: `
v:
- A
- B
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "\nv",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.SequenceEntryType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         "-",
					Origin:        "\n-",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "A",
					Origin:        " A\n",
				},
				{
					Type:          token.SequenceEntryType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         "-",
					Origin:        "-",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "B",
					Origin:        " B",
				},
			},
		},
		{
			YAML: `
v:
- A
- |-
 B
 C
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "\nv",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.SequenceEntryType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         "-",
					Origin:        "\n-",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "A",
					Origin:        " A\n",
				},
				{
					Type:          token.SequenceEntryType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         "-",
					Origin:        "-",
				},
				{
					Type:          token.LiteralType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockScalarIndicator,
					Value:         "|-",
					Origin:        " |-\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "B\nC",
					Origin:        " B\n C\n",
				},
			},
		},
		{
			YAML: `
v:
- A
- 1
- B:
 - 2
 - 3
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "v",
					Origin:        "\nv",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.SequenceEntryType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         "-",
					Origin:        "\n-",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "A",
					Origin:        " A\n",
				},
				{
					Type:          token.SequenceEntryType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         "-",
					Origin:        "-",
				},
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "1",
					Origin:        " 1\n",
				},
				{
					Type:          token.SequenceEntryType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         "-",
					Origin:        "-",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "B",
					Origin:        " B",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.SequenceEntryType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         "-",
					Origin:        "\n -",
				},
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "2",
					Origin:        " 2\n ",
				},
				{
					Type:          token.SequenceEntryType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         "-",
					Origin:        "-",
				},
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "3",
					Origin:        " 3",
				},
			},
		},
		{
			YAML: `
a:
 b: c
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "\na",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "b",
					Origin:        "\n b",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "c",
					Origin:        " c",
				},
			},
		},
		{
			YAML: `a: '-'`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.SingleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "-",
					Origin:        " '-'",
				},
			},
		},
		{
			YAML: `123`,
			Tokens: token.Tokens{
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "123",
					Origin:        "123",
				},
			},
		},
		{
			YAML: `hello: world
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "hello",
					Origin:        "hello",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "world",
					Origin:        " world",
				},
			},
		},
		{
			YAML: `a: null`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.NullType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "null",
					Origin:        " null",
				},
			},
		},
		{
			YAML: `a: {x: 1}`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.MappingStartType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.FlowCollectionIndicator,
					Value:         "{",
					Origin:        " {",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "x",
					Origin:        "x",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "1",
					Origin:        " 1",
				},
				{
					Type:          token.MappingEndType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.FlowCollectionIndicator,
					Value:         "}",
					Origin:        "}",
				},
			},
		},
		{
			YAML: `a: [1, 2]`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.SequenceStartType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.FlowCollectionIndicator,
					Value:         "[",
					Origin:        " [",
				},
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "1",
					Origin:        "1",
				},
				{
					Type:          token.CollectEntryType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.FlowCollectionIndicator,
					Value:         ",",
					Origin:        ",",
				},
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "2",
					Origin:        " 2",
				},
				{
					Type:          token.SequenceEndType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.FlowCollectionIndicator,
					Value:         "]",
					Origin:        "]",
				},
			},
		},
		{
			YAML: `
t2: 2018-01-09T10:40:47Z
t4: 2098-01-09T10:40:47Z
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "t2",
					Origin:        "\nt2",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "2018-01-09T10:40:47Z",
					Origin:        " 2018-01-09T10:40:47Z\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "t4",
					Origin:        "t4",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "2098-01-09T10:40:47Z",
					Origin:        " 2098-01-09T10:40:47Z",
				},
			},
		},
		{
			YAML: `a: {b: c, d: e}`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.MappingStartType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.FlowCollectionIndicator,
					Value:         "{",
					Origin:        " {",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "b",
					Origin:        "b",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "c",
					Origin:        " c",
				},
				{
					Type:          token.CollectEntryType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.FlowCollectionIndicator,
					Value:         ",",
					Origin:        ",",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "d",
					Origin:        " d",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "e",
					Origin:        " e",
				},
				{
					Type:          token.MappingEndType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.FlowCollectionIndicator,
					Value:         "}",
					Origin:        "}",
				},
			},
		},
		{
			YAML: `a: 3s`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "3s",
					Origin:        " 3s",
				},
			},
		},
		{
			YAML: `a: <foo>`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "<foo>",
					Origin:        " <foo>",
				},
			},
		},
		{
			YAML: `a: "1:1"`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.DoubleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "1:1",
					Origin:        " \"1:1\"",
				},
			},
		},
		{
			YAML: `a: "\0"`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.DoubleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "\x00",
					Origin:        " \"\\0\"",
				},
			},
		},
		{
			YAML: `a: !!binary gIGC`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.TagType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.NodePropertyIndicator,
					Value:         "!!binary",
					Origin:        " !!binary ",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "gIGC",
					Origin:        "gIGC",
				},
			},
		},
		{
			YAML: `
a: !!binary |
 kJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJ
 CQ
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "\na",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.TagType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.NodePropertyIndicator,
					Value:         "!!binary",
					Origin:        " !!binary ",
				},
				{
					Type:          token.LiteralType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockScalarIndicator,
					Value:         "|",
					Origin:        "|\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "kJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJ\nCQ\n",
					Origin:        " kJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJ\n CQ\n",
				},
			},
		},
		{
			YAML: `
b: 2
a: 1
d: 4
c: 3
sub:
  e: 5
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "b",
					Origin:        "\nb",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "2",
					Origin:        " 2\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "1",
					Origin:        " 1\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "d",
					Origin:        "d",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "4",
					Origin:        " 4\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "c",
					Origin:        "c",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "3",
					Origin:        " 3\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "sub",
					Origin:        "sub",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "e",
					Origin:        "\n  e",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.IntegerType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "5",
					Origin:        " 5",
				},
			},
		},
		{
			YAML: `a: 1.2.3.4`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "1.2.3.4",
					Origin:        " 1.2.3.4",
				},
			},
		},
		{
			YAML: `a: "2015-02-24T18:19:39Z"`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.DoubleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "2015-02-24T18:19:39Z",
					Origin:        " \"2015-02-24T18:19:39Z\"",
				},
			},
		},
		{
			YAML: `a: 'b: c'`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.SingleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "b: c",
					Origin:        " 'b: c'",
				},
			},
		},
		{
			YAML: `a: 'Hello #comment'`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.SingleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "Hello #comment",
					Origin:        " 'Hello #comment'",
				},
			},
		},
		{
			YAML: `a: 100.5`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.FloatType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "100.5",
					Origin:        " 100.5",
				},
			},
		},
		{
			YAML: `a: bogus`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "a",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "bogus",
					Origin:        " bogus",
				},
			},
		},
		{
			YAML: `"a": double quoted map key`,
			Tokens: token.Tokens{
				{
					Type:          token.DoubleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "a",
					Origin:        "\"a\"",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "double quoted map key",
					Origin:        " double quoted map key",
				},
			},
		},
		{
			YAML: `'a': single quoted map key`,
			Tokens: token.Tokens{
				{
					Type:          token.SingleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "a",
					Origin:        "'a'",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "single quoted map key",
					Origin:        " single quoted map key",
				},
			},
		},
		{
			YAML: `
a: "double quoted"
b: "value map"`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "\na",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.DoubleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "double quoted",
					Origin:        " \"double quoted\"",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "b",
					Origin:        "\nb",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.DoubleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "value map",
					Origin:        " \"value map\"",
				},
			},
		},
		{
			YAML: `
a: 'single quoted'
b: 'value map'`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "\na",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.SingleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "single quoted",
					Origin:        " 'single quoted'",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "b",
					Origin:        "\nb",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.SingleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "value map",
					Origin:        " 'value map'",
				},
			},
		},
		{
			YAML: `json: '\"expression\": \"thi:\"'`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "json",
					Origin:        "json",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.SingleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "\\\"expression\\\": \\\"thi:\\\"",
					Origin:        " '\\\"expression\\\": \\\"thi:\\\"'",
				},
			},
		},
		{
			YAML: `json: "\"expression\": \"thi:\""`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "json",
					Origin:        "json",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.DoubleQuoteType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.QuotedScalarIndicator,
					Value:         "\"expression\": \"thi:\"",
					Origin:        " \"\\\"expression\\\": \\\"thi:\\\"\"",
				},
			},
		},
		{
			YAML: `
a:
 b

 c
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "\na",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "b\nc",
					Origin:        "\n b\n\n c",
				},
			},
		},
		{
			YAML: `
a:   
 b   

  
 c
 d 
e: f
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "\na",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "b\nc d",
					Origin:        "\n b\n\n\n c\n d\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "e",
					Origin:        "e",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "f",
					Origin:        " f",
				},
			},
		},
		{
			YAML: `
a: |
 b   

  
 c
 d 
e: f
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "\na",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.LiteralType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockScalarIndicator,
					Value:         "|",
					Origin:        " |\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "b   \n\n \nc\nd \n",
					Origin:        " b   \n\n  \n c\n d \n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "e",
					Origin:        "e",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "f",
					Origin:        " f",
				},
			},
		},
		{
			YAML: `
a: >
 b   

  
 c
 d 
e: f
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "\na",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.FoldedType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockScalarIndicator,
					Value:         ">",
					Origin:        " >\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "b   \n\n \nc d \n",
					Origin:        " b   \n\n  \n c\n d \n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "e",
					Origin:        "e",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "f",
					Origin:        " f",
				},
			},
		},
		{
			YAML: `
a: >
  Text`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a",
					Origin:        "\na",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.FoldedType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockScalarIndicator,
					Value:         ">",
					Origin:        " >\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "Text",
					Origin:        "  Text",
				},
			},
		},
		{
			YAML: `
s: >
        1s
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "s",
					Origin:        "\ns",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.FoldedType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockScalarIndicator,
					Value:         ">",
					Origin:        " >\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "1s\n",
					Origin:        "        1s\n",
				},
			},
		},
		{
			YAML: `
s: >1        # comment
        1s
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "s",
					Origin:        "\ns",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.FoldedType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockScalarIndicator,
					Value:         ">1",
					Origin:        " >1        ",
				},
				{
					Type:          token.CommentType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.CommentIndicator,
					Value:         " comment",
					Origin:        "# comment\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "       1s\n",
					Origin:        "        1s\n",
				},
			},
		},
		{
			YAML: `
s: >+2
        1s
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "s",
					Origin:        "\ns",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.FoldedType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockScalarIndicator,
					Value:         ">+2",
					Origin:        " >+2\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "      1s\n",
					Origin:        "        1s\n",
				},
			},
		},
		{
			YAML: `
s: >-3
        1s
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "s",
					Origin:        "\ns",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.FoldedType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockScalarIndicator,
					Value:         ">-3",
					Origin:        " >-3\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "     1s",
					Origin:        "        1s\n",
				},
			},
		},
		{
			YAML: `
s: >
    1s
    2s
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "s",
					Origin:        "\ns",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.FoldedType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockScalarIndicator,
					Value:         ">",
					Origin:        " >\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "1s 2s\n",
					Origin:        "    1s\n    2s\n",
				},
			},
		},
		{
			YAML: `
s: >
    1s
      2s
    3s
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "s",
					Origin:        "\ns",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.FoldedType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockScalarIndicator,
					Value:         ">",
					Origin:        " >\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "1s\n  2s\n3s\n",
					Origin:        "    1s\n      2s\n    3s\n",
				},
			},
		},
		{
			YAML: `
s: >
    1s
      2s
      3s
    4s
    5s
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "s",
					Origin:        "\ns",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.FoldedType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockScalarIndicator,
					Value:         ">",
					Origin:        " >\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "1s\n  2s\n  3s\n4s 5s\n",
					Origin:        "    1s\n      2s\n      3s\n    4s\n    5s\n",
				},
			},
		},
		{
			YAML: `
s: >-3
    1s
      2s
      3s
    4s
    5s
`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "s",
					Origin:        "\ns",
				},
				{
					Type:          token.MappingValueType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockStructureIndicator,
					Value:         ":",
					Origin:        ":",
				},
				{
					Type:          token.FoldedType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockScalarIndicator,
					Value:         ">-3",
					Origin:        " >-3\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         " 1s\n   2s\n   3s\n 4s\n 5s",
					Origin:        "    1s\n      2s\n      3s\n    4s\n    5s\n",
				},
			},
		},
		{
			YAML: `
|2-

                  text
`,
			Tokens: token.Tokens{
				{
					Type:          token.LiteralType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockScalarIndicator,
					Value:         "|2-",
					Origin:        "\n|2-\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "\n                text",
					Origin:        "\n                  text\n",
				},
			},
		},
		{
			YAML: `
|
  a



`,
			Tokens: token.Tokens{
				{
					Type:          token.LiteralType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockScalarIndicator,
					Value:         "|",
					Origin:        "\n|\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "a\n",
					Origin:        "  a\n\n\n\n",
				},
			},
		},
		{
			YAML: `
|  		  # comment
  foo
`,
			Tokens: token.Tokens{
				{
					Type:          token.LiteralType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.BlockScalarIndicator,
					Value:         "|",
					Origin:        "\n|  		  ",
				},
				{
					Type:          token.CommentType,
					CharacterType: token.CharacterTypeIndicator,
					Indicator:     token.CommentIndicator,
					Value:         " comment",
					Origin:        "# comment\n",
				},
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "foo\n",
					Origin:        "  foo\n",
				},
			},
		},
		{
			YAML: `1x0`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "1x0",
					Origin:        "1x0",
				},
			},
		},
		{
			YAML: `0b98765`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "0b98765",
					Origin:        "0b98765",
				},
			},
		},
		{
			YAML: `098765`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "098765",
					Origin:        "098765",
				},
			},
		},
		{
			YAML: `0o98765`,
			Tokens: token.Tokens{
				{
					Type:          token.StringType,
					CharacterType: token.CharacterTypeMiscellaneous,
					Indicator:     token.NotIndicator,
					Value:         "0o98765",
					Origin:        "0o98765",
				},
			},
		},
	}
	for _, test := range tests {
		t.Run(test.YAML, func(t *testing.T) {
			tokens := lexer.Tokenize(test.YAML)
			if len(tokens) != len(test.Tokens) {
				t.Fatalf("Tokenize(%q) token count mismatch, expected: %d got: %d", test.YAML, len(test.Tokens), len(tokens))
			}
			for i := range test.Tokens {
				if tokens[i].Type != test.Tokens[i].Type {
					t.Errorf("Tokenize(%q)[%d] token.Type mismatch, expected: %s got: %s", test.YAML, i, test.Tokens[i].Type, tokens[i].Type)
				}
				if tokens[i].CharacterType != test.Tokens[i].CharacterType {
					t.Errorf("Tokenize(%q)[%d] token.CharacterType mismatch, expected: %s got: %s", test.YAML, i, test.Tokens[i].CharacterType, tokens[i].CharacterType)
				}
				if tokens[i].Indicator != test.Tokens[i].Indicator {
					t.Errorf("Tokenize(%q)[%d] token.Indicator mismatch, expected: %s got: %s", test.YAML, i, test.Tokens[i].Indicator, tokens[i].Indicator)
				}
				if tokens[i].Value != test.Tokens[i].Value {
					t.Errorf("Tokenize(%q)[%d] token.Value mismatch, expected: %q got: %q", test.YAML, i, test.Tokens[i].Value, tokens[i].Value)
				}
				if tokens[i].Origin != test.Tokens[i].Origin {
					t.Errorf("Tokenize(%q)[%d] token.Origin mismatch, expected: %q got: %q", test.YAML, i, test.Tokens[i].Origin, tokens[i].Origin)
				}
			}
		})
	}
}

type testToken struct {
	line   int
	column int
	value  string
}

func TestSingleLineToken_ValueLineColumnPosition(t *testing.T) {
	tests := []struct {
		name   string
		src    string
		expect map[int]string // Column -> Value map.
	}{
		{
			name: "single quote, single value array",
			src:  "test: ['test']",
			expect: map[int]string{
				1:  "test",
				5:  ":",
				7:  "[",
				8:  "test",
				14: "]",
			},
		},
		{
			name: "double quote, single value array",
			src:  `test: ["test"]`,
			expect: map[int]string{
				1:  "test",
				5:  ":",
				7:  "[",
				8:  "test",
				14: "]",
			},
		},
		{
			name: "no quotes, single value array",
			src:  "test: [somevalue]",
			expect: map[int]string{
				1:  "test",
				5:  ":",
				7:  "[",
				8:  "somevalue",
				17: "]",
			},
		},
		{
			name: "single quote, multi value array",
			src:  "myarr: ['1','2','3', '444' , '55','66' ,  '77'  ]",
			expect: map[int]string{
				1:  "myarr",
				6:  ":",
				8:  "[",
				9:  "1",
				12: ",",
				13: "2",
				16: ",",
				17: "3",
				20: ",",
				22: "444",
				28: ",",
				30: "55",
				34: ",",
				35: "66",
				40: ",",
				43: "77",
				49: "]",
			},
		},
		{
			name: "double quote, multi value array",
			src:  `myarr: ["1","2","3", "444" , "55","66" ,  "77"  ]`,
			expect: map[int]string{
				1:  "myarr",
				6:  ":",
				8:  "[",
				9:  "1",
				12: ",",
				13: "2",
				16: ",",
				17: "3",
				20: ",",
				22: "444",
				28: ",",
				30: "55",
				34: ",",
				35: "66",
				40: ",",
				43: "77",
				49: "]",
			},
		},
		{
			name: "no quote, multi value array",
			src:  "numbers: [1, 5, 99,100, 3, 7 ]",
			expect: map[int]string{
				1:  "numbers",
				8:  ":",
				10: "[",
				11: "1",
				12: ",",
				14: "5",
				15: ",",
				17: "99",
				19: ",",
				20: "100",
				23: ",",
				25: "3",
				26: ",",
				28: "7",
				30: "]",
			},
		},
		{
			name: "double quotes, nested arrays",
			src:  `Strings: ["1",["2",["3"]]]`,
			expect: map[int]string{
				1:  "Strings",
				8:  ":",
				10: "[",
				11: "1",
				14: ",",
				15: "[",
				16: "2",
				19: ",",
				20: "[",
				21: "3",
				24: "]",
				25: "]",
				26: "]",
			},
		},
		{
			name: "mixed quotes, nested arrays",
			src:  `Values: [1,['2',"3",4,["5",6]]]`,
			expect: map[int]string{
				1:  "Values",
				7:  ":",
				9:  "[",
				10: "1",
				11: ",",
				12: "[",
				13: "2",
				16: ",",
				17: "3",
				20: ",",
				21: "4",
				22: ",",
				23: "[",
				24: "5",
				27: ",",
				28: "6",
				29: "]",
				30: "]",
				31: "]",
			},
		},
		{
			name: "double quote, empty array",
			src:  `Empty: ["", ""]`,
			expect: map[int]string{
				1:  "Empty",
				6:  ":",
				8:  "[",
				9:  "",
				11: ",",
				13: "",
				15: "]",
			},
		},
		{
			name: "double quote key",
			src:  `"a": b`,
			expect: map[int]string{
				1: "a",
				4: ":",
				6: "b",
			},
		},
		{
			name: "single quote key",
			src:  `'a': b`,
			expect: map[int]string{
				1: "a",
				4: ":",
				6: "b",
			},
		},
		{
			name: "double quote key and value",
			src:  `"a": "b"`,
			expect: map[int]string{
				1: "a",
				4: ":",
				6: "b",
			},
		},
		{
			name: "single quote key and value",
			src:  `'a': 'b'`,
			expect: map[int]string{
				1: "a",
				4: ":",
				6: "b",
			},
		},
		{
			name: "double quote key, single quote value",
			src:  `"a": 'b'`,
			expect: map[int]string{
				1: "a",
				4: ":",
				6: "b",
			},
		},
		{
			name: "single quote key, double quote value",
			src:  `'a': "b"`,
			expect: map[int]string{
				1: "a",
				4: ":",
				6: "b",
			},
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			got := lexer.Tokenize(tc.src)
			sort.Slice(got, func(i, j int) bool {
				return got[i].Position.Column < got[j].Position.Column
			})
			var expected []testToken
			for k, v := range tc.expect {
				tt := testToken{
					line:   1,
					column: k,
					value:  v,
				}
				expected = append(expected, tt)
			}
			sort.Slice(expected, func(i, j int) bool {
				return expected[i].column < expected[j].column
			})
			if len(got) != len(expected) {
				t.Errorf("Tokenize(%s) token count mismatch, expected:%d got:%d", tc.src, len(expected), len(got))
			}
			for i, tok := range got {
				if !tokenMatches(tok, expected[i]) {
					t.Errorf("Tokenize(%s) expected:%+v got line:%d column:%d value:%s", tc.src, expected[i], tok.Position.Line, tok.Position.Column, tok.Value)
				}
			}
		})
	}
}

func tokenMatches(t *token.Token, e testToken) bool {
	return t != nil && t.Position != nil &&
		t.Value == e.value &&
		t.Position.Line == e.line &&
		t.Position.Column == e.column
}

func TestMultiLineToken_ValueLineColumnPosition(t *testing.T) {
	tests := []struct {
		name   string
		src    string
		expect []testToken
	}{
		{
			name: "double quote",
			src: `one: "1 2 3 4 5"
two: "1 2
3 4
5"
three: "1 2 3 4
5"`,
			expect: []testToken{
				{
					line:   1,
					column: 1,
					value:  "one",
				},
				{
					line:   1,
					column: 4,
					value:  ":",
				},
				{
					line:   1,
					column: 6,
					value:  "1 2 3 4 5",
				},
				{
					line:   2,
					column: 1,
					value:  "two",
				},
				{
					line:   2,
					column: 4,
					value:  ":",
				},
				{
					line:   2,
					column: 6,
					value:  "1 2 3 4 5",
				},
				{
					line:   5,
					column: 1,
					value:  "three",
				},
				{
					line:   5,
					column: 6,
					value:  ":",
				},
				{
					line:   5,
					column: 8,
					value:  "1 2 3 4 5",
				},
			},
		},
		{
			name: "single quote in an array",
			src: `arr: ['1', 'and
two']
last: 'hello'`,
			expect: []testToken{
				{
					line:   1,
					column: 1,
					value:  "arr",
				},
				{
					line:   1,
					column: 4,
					value:  ":",
				},
				{
					line:   1,
					column: 6,
					value:  "[",
				},
				{
					line:   1,
					column: 7,
					value:  "1",
				},
				{
					line:   1,
					column: 10,
					value:  ",",
				},
				{
					line:   1,
					column: 12,
					value:  "and two",
				},
				{
					line:   2,
					column: 5,
					value:  "]",
				},
				{
					line:   3,
					column: 1,
					value:  "last",
				},
				{
					line:   3,
					column: 5,
					value:  ":",
				},
				{
					line:   3,
					column: 7,
					value:  "hello",
				},
			},
		},
		{
			name: "single quote and double quote",
			src: `foo: "test




bar"
foo2: 'bar2'`,
			expect: []testToken{
				{
					line:   1,
					column: 1,
					value:  "foo",
				},
				{
					line:   1,
					column: 4,
					value:  ":",
				},
				{
					line:   1,
					column: 6,
					value:  "test\n\n\n\nbar",
				},
				{
					line:   7,
					column: 1,
					value:  "foo2",
				},
				{
					line:   7,
					column: 5,
					value:  ":",
				},
				{
					line:   7,
					column: 7,
					value:  "bar2",
				},
			},
		},
		{
			name: "single and double quote map keys",
			src: `"a": test
'b': 1
c: true`,
			expect: []testToken{
				{
					line:   1,
					column: 1,
					value:  "a",
				},
				{
					line:   1,
					column: 4,
					value:  ":",
				},
				{
					line:   1,
					column: 6,
					value:  "test",
				},
				{
					line:   2,
					column: 1,
					value:  "b",
				},
				{
					line:   2,
					column: 4,
					value:  ":",
				},
				{
					line:   2,
					column: 6,
					value:  "1",
				},
				{
					line:   3,
					column: 1,
					value:  "c",
				},
				{
					line:   3,
					column: 2,
					value:  ":",
				},
				{
					line:   3,
					column: 4,
					value:  "true",
				},
			},
		},
		{
			name: "issue326",
			src: `a: |
  Text
b: 1`,
			expect: []testToken{
				{
					line:   1,
					column: 1,
					value:  "a",
				},
				{
					line:   1,
					column: 2,
					value:  ":",
				},
				{
					line:   1,
					column: 4,
					value:  "|",
				},
				{
					line:   2,
					column: 3,
					value:  "Text\n",
				},
				{
					line:   3,
					column: 1,
					value:  "b",
				},
				{
					line:   3,
					column: 2,
					value:  ":",
				},
				{
					line:   3,
					column: 4,
					value:  "1",
				},
			},
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			got := lexer.Tokenize(tc.src)
			sort.Slice(got, func(i, j int) bool {
				// sort by line, then column
				if got[i].Position.Line < got[j].Position.Line {
					return true
				} else if got[i].Position.Line == got[j].Position.Line {
					return got[i].Position.Column < got[j].Position.Column
				}
				return false
			})
			sort.Slice(tc.expect, func(i, j int) bool {
				if tc.expect[i].line < tc.expect[j].line {
					return true
				} else if tc.expect[i].line == tc.expect[j].line {
					return tc.expect[i].column < tc.expect[j].column
				}
				return false
			})
			if len(got) != len(tc.expect) {
				t.Errorf("Tokenize() token count mismatch, expected:%d got:%d", len(tc.expect), len(got))
			}
			for i, tok := range got {
				if !tokenMatches(tok, tc.expect[i]) {
					t.Errorf("Tokenize() expected:%+v got line:%d column:%d value:%s", tc.expect[i], tok.Position.Line, tok.Position.Column, tok.Value)
				}
			}
		})
	}
}

func TestInvalid(t *testing.T) {
	tests := []struct {
		name string
		src  string
	}{
		{
			name: "literal opt with content",
			src: `
a: |invalid
  foo`,
		},
		{
			name: "literal opt",
			src: `
a: |invalid`,
		},
		{
			name: "invalid single-quoted",
			src:  `a: 'foobarbaz`,
		},
		{
			name: "invalid double-quoted",
			src:  `a: "\"key\": \"value:\"`,
		},
		{
			name: "invalid document folded",
			src:  ">\n>",
		},
		{
			name: "invalid document number",
			src:  ">\n1",
		},
		{
			name: "invalid document header option number",
			src:  "a: >3\n  1",
		},
		{
			name: "use reserved character @",
			src:  "key: [@val]",
		},
		{
			name: "use reserved character `",
			src:  "key: [`val]",
		},
		{
			name: "use tab character as indent",
			// nolint: gci
			src: "	a: b",
		},
		{
			name: "use tab character as indent in literal",
			src: `
a: |
	b
	c
`,
		},
		{
			name: "invalid UTF-16 character",
			src:  `"\u00"`,
		},
		{
			name: "invalid UTF-16 surrogate pair length",
			src:  `"\ud800"`,
		},
		{
			name: "invalid UTF-16 low surrogate prefix",
			src:  `"\ud800\v"`,
		},
		{
			name: "invalid UTF-16 low surrogate",
			src:  `"\ud800\u0000"`,
		},
		{
			name: "invalid UTF-32 character",
			src:  `"\U0000"`,
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			got := lexer.Tokenize(test.src)
			if got.InvalidToken() == nil {
				t.Fatal("expected contains invalid token")
			}
		})
	}
}

func TestTokenOffset(t *testing.T) {
	t.Run("crlf", func(t *testing.T) {
		content := "project:\r\n  version: 1.2.3\r\n"
		tokens := lexer.Tokenize(content)
		if len(tokens) != 5 {
			t.Fatalf("invalid token num. got %d", len(tokens))
		}
		if tokens[4].Value != "1.2.3" {
			t.Fatalf("unexpected value. got %q", tokens[4].Value)
		}
		if tokens[4].Position.Offset != 22 {
			t.Fatalf("unexpected offset. got %d", tokens[4].Position.Offset)
		}
	})
	t.Run("lf", func(t *testing.T) {
		content := "project:\n  version: 1.2.3\n"
		tokens := lexer.Tokenize(content)
		if len(tokens) != 5 {
			t.Fatalf("invalid token num. got %d", len(tokens))
		}
		if tokens[4].Value != "1.2.3" {
			t.Fatalf("unexpected value. got %q", tokens[4].Value)
		}
		if tokens[4].Position.Offset != 21 {
			t.Fatalf("unexpected offset. got %d", tokens[4].Position.Offset)
		}
	})
}