Stowage/go-yaml
2
0
Fork 0
mirror of https://github.com/goccy/go-yaml.git synced 2025-11-01 05:41:14 +00:00
Code Issues Projects Releases Packages Wiki Activity

Support parsing with comment

Browse source
This commit is contained in:
Masaaki Goshima 2020-05-28 21:39:06 +09:00
parent 05f465a371
commit b3d4080492
4 changed files with 461 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

306
ast/ast.go
View file

@ -7,6 +7,11 @@ import (
"strings"
"github.com/goccy/go-yaml/token"
"golang.org/x/xerrors"
)
var (
ErrInvalidTokenType = xerrors.New("invalid token type")
)
// NodeType type identifier of node
@ -49,6 +54,8 @@ const (
DirectiveType
// TagType type identifier for tag node
TagType
// CommentType type identifier for comment node
CommentType
)
// String node type identifier to text
@ -90,6 +97,8 @@ func (t NodeType) String() string {
return "Directive"
case TagType:
return "Tag"
case CommentType:
return "Comment"
}
return ""
}
@ -104,6 +113,10 @@ type Node interface {
Type() NodeType
// AddColumn add column number to child nodes recursively
AddColumn(int)
// SetComment set comment token to node
SetComment(*token.Token) error
// Comment returns comment token instance
GetComment() *token.Token
}
// File contains all documents in YAML file
@ -123,6 +136,7 @@ func (f *File) String() string {
// Document type of Document
type Document struct {
Comment *token.Token // position of Comment ( `#comment` )
Start *token.Token // position of DocumentHeader ( `---` )
End *token.Token // position of DocumentEnd ( `...` )
Body Node
@ -133,6 +147,11 @@ func (d *Document) GetToken() *token.Token {
return d.Body.GetToken()
}
// GetComment returns comment token instance
func (d *Document) GetComment() *token.Token {
return d.Comment
}
// AddColumn add column number to child nodes recursively
func (d *Document) AddColumn(col int) {
if d.Body != nil {
@ -140,6 +159,15 @@ func (d *Document) AddColumn(col int) {
}
}
// SetComment set comment token
func (d *Document) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
d.Comment = tk
return nil
}
// Type returns DocumentType
func (d *Document) Type() NodeType { return DocumentType }
@ -280,6 +308,11 @@ func String(tk *token.Token) Node {
}
}
// Comment create node for comment
func Comment(tk *token.Token) Node {
return &CommentNode{Comment: tk}
}
// MergeKey create node for merge key ( << )
func MergeKey(tk *token.Token) Node {
return &MergeKeyNode{
@ -308,6 +341,7 @@ func Sequence(tk *token.Token, isFlowStyle bool) *SequenceNode {
// NullNode type of null node
type NullNode struct {
ScalarNode
Comment *token.Token // position of Comment ( `#comment` )
Token *token.Token
}
@ -319,11 +353,25 @@ func (n *NullNode) GetToken() *token.Token {
return n.Token
}
// GetComment returns comment token instance
func (n *NullNode) GetComment() *token.Token {
return n.Comment
}
// AddColumn add column number to child nodes recursively
func (n *NullNode) AddColumn(col int) {
n.Token.AddColumn(col)
}
// SetComment set comment token
func (n *NullNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
// GetValue returns nil value
func (n *NullNode) GetValue() interface{} {
return nil
@ -337,6 +385,7 @@ func (n *NullNode) String() string {
// IntegerNode type of integer node
type IntegerNode struct {
ScalarNode
Comment *token.Token // position of Comment ( `#comment` )
Token *token.Token
Value interface{} // int64 or uint64 value
}
@ -349,11 +398,25 @@ func (n *IntegerNode) GetToken() *token.Token {
return n.Token
}
// GetComment returns comment token instance
func (n *IntegerNode) GetComment() *token.Token {
return n.Comment
}
// AddColumn add column number to child nodes recursively
func (n *IntegerNode) AddColumn(col int) {
n.Token.AddColumn(col)
}
// SetComment set comment token
func (n *IntegerNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
// GetValue returns int64 value
func (n *IntegerNode) GetValue() interface{} {
return n.Value
@ -367,6 +430,7 @@ func (n *IntegerNode) String() string {
// FloatNode type of float node
type FloatNode struct {
ScalarNode
Comment *token.Token // position of Comment ( `#comment` )
Token *token.Token
Precision int
Value float64
@ -380,11 +444,25 @@ func (n *FloatNode) GetToken() *token.Token {
return n.Token
}
// GetComment returns comment token instance
func (n *FloatNode) GetComment() *token.Token {
return n.Comment
}
// AddColumn add column number to child nodes recursively
func (n *FloatNode) AddColumn(col int) {
n.Token.AddColumn(col)
}
// SetComment set comment token
func (n *FloatNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
// GetValue returns float64 value
func (n *FloatNode) GetValue() interface{} {
return n.Value
@ -398,6 +476,7 @@ func (n *FloatNode) String() string {
// StringNode type of string node
type StringNode struct {
ScalarNode
Comment *token.Token // position of Comment ( `#comment` )
Token *token.Token
Value string
}
@ -410,11 +489,25 @@ func (n *StringNode) GetToken() *token.Token {
return n.Token
}
// GetComment returns comment token instance
func (n *StringNode) GetComment() *token.Token {
return n.Comment
}
// AddColumn add column number to child nodes recursively
func (n *StringNode) AddColumn(col int) {
n.Token.AddColumn(col)
}
// SetComment set comment token
func (n *StringNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
// GetValue returns string value
func (n *StringNode) GetValue() interface{} {
return n.Value
@ -450,6 +543,7 @@ func (n *StringNode) String() string {
// LiteralNode type of literal node
type LiteralNode struct {
ScalarNode
Comment *token.Token // position of Comment ( `#comment` )
Start *token.Token
Value *StringNode
}
@ -462,6 +556,11 @@ func (n *LiteralNode) GetToken() *token.Token {
return n.Start
}
// GetComment returns comment token instance
func (n *LiteralNode) GetComment() *token.Token {
return n.Comment
}
// AddColumn add column number to child nodes recursively
func (n *LiteralNode) AddColumn(col int) {
n.Start.AddColumn(col)
@ -470,6 +569,15 @@ func (n *LiteralNode) AddColumn(col int) {
}
}
// SetComment set comment token
func (n *LiteralNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
// GetValue returns string value
func (n *LiteralNode) GetValue() interface{} {
return n.String()
@ -484,6 +592,7 @@ func (n *LiteralNode) String() string {
// MergeKeyNode type of merge key node
type MergeKeyNode struct {
ScalarNode
Comment *token.Token // position of Comment ( `#comment` )
Token *token.Token
}
@ -495,6 +604,11 @@ func (n *MergeKeyNode) GetToken() *token.Token {
return n.Token
}
// GetComment returns comment token instance
func (n *MergeKeyNode) GetComment() *token.Token {
return n.Comment
}
// GetValue returns '<<' value
func (n *MergeKeyNode) GetValue() interface{} {
return n.Token.Value
@ -510,9 +624,19 @@ func (n *MergeKeyNode) AddColumn(col int) {
n.Token.AddColumn(col)
}
// SetComment set comment token
func (n *MergeKeyNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
// BoolNode type of boolean node
type BoolNode struct {
ScalarNode
Comment *token.Token // position of Comment ( `#comment` )
Token *token.Token
Value bool
}
@ -525,11 +649,25 @@ func (n *BoolNode) GetToken() *token.Token {
return n.Token
}
// GetComment returns comment token instance
func (n *BoolNode) GetComment() *token.Token {
return n.Comment
}
// AddColumn add column number to child nodes recursively
func (n *BoolNode) AddColumn(col int) {
n.Token.AddColumn(col)
}
// SetComment set comment token
func (n *BoolNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
// GetValue returns boolean value
func (n *BoolNode) GetValue() interface{} {
return n.Value
@ -543,6 +681,7 @@ func (n *BoolNode) String() string {
// InfinityNode type of infinity node
type InfinityNode struct {
ScalarNode
Comment *token.Token // position of Comment ( `#comment` )
Token *token.Token
Value float64
}
@ -555,11 +694,25 @@ func (n *InfinityNode) GetToken() *token.Token {
return n.Token
}
// GetComment returns comment token instance
func (n *InfinityNode) GetComment() *token.Token {
return n.Comment
}
// AddColumn add column number to child nodes recursively
func (n *InfinityNode) AddColumn(col int) {
n.Token.AddColumn(col)
}
// SetComment set comment token
func (n *InfinityNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
// GetValue returns math.Inf(0) or math.Inf(-1)
func (n *InfinityNode) GetValue() interface{} {
return n.Value
@ -573,6 +726,7 @@ func (n *InfinityNode) String() string {
// NanNode type of nan node
type NanNode struct {
ScalarNode
Comment *token.Token // position of Comment ( `#comment` )
Token *token.Token
}
@ -584,11 +738,25 @@ func (n *NanNode) GetToken() *token.Token {
return n.Token
}
// GetComment returns comment token instance
func (n *NanNode) GetComment() *token.Token {
return n.Comment
}
// AddColumn add column number to child nodes recursively
func (n *NanNode) AddColumn(col int) {
n.Token.AddColumn(col)
}
// SetComment set comment token
func (n *NanNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
// GetValue returns math.NaN()
func (n *NanNode) GetValue() interface{} {
return math.NaN()
@ -634,6 +802,7 @@ func (m *MapNodeIter) Value() Node {
// MappingNode type of mapping node
type MappingNode struct {
Comment *token.Token // position of Comment ( `#comment` )
Start *token.Token
End *token.Token
IsFlowStyle bool
@ -648,6 +817,11 @@ func (n *MappingNode) GetToken() *token.Token {
return n.Start
}
// GetComment returns comment token instance
func (n *MappingNode) GetComment() *token.Token {
return n.Comment
}
// AddColumn add column number to child nodes recursively
func (n *MappingNode) AddColumn(col int) {
n.Start.AddColumn(col)
@ -657,6 +831,15 @@ func (n *MappingNode) AddColumn(col int) {
}
}
// SetComment set comment token
func (n *MappingNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
func (n *MappingNode) flowStyleString() string {
if len(n.Values) == 0 {
return "{}"
@ -697,6 +880,7 @@ func (n *MappingNode) MapRange() *MapNodeIter {
// MappingValueNode type of mapping value
type MappingValueNode struct {
Comment *token.Token // position of Comment ( `#comment` )
Start *token.Token
Key Node
Value Node
@ -710,6 +894,11 @@ func (n *MappingValueNode) GetToken() *token.Token {
return n.Start
}
// GetComment returns comment token instance
func (n *MappingValueNode) GetComment() *token.Token {
return n.Comment
}
// AddColumn add column number to child nodes recursively
func (n *MappingValueNode) AddColumn(col int) {
n.Start.AddColumn(col)
@ -721,6 +910,15 @@ func (n *MappingValueNode) AddColumn(col int) {
}
}
// SetComment set comment token
func (n *MappingValueNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
// String mapping value to text
func (n *MappingValueNode) String() string {
space := strings.Repeat(" ", n.Key.GetToken().Position.Column-1)
@ -781,6 +979,7 @@ func (m *ArrayNodeIter) Len() int {
// SequenceNode type of sequence node
type SequenceNode struct {
Comment *token.Token // position of Comment ( `#comment` )
Start *token.Token
End *token.Token
IsFlowStyle bool
@ -795,6 +994,11 @@ func (n *SequenceNode) GetToken() *token.Token {
return n.Start
}
// GetComment returns comment token instance
func (n *SequenceNode) GetComment() *token.Token {
return n.Comment
}
// AddColumn add column number to child nodes recursively
func (n *SequenceNode) AddColumn(col int) {
n.Start.AddColumn(col)
@ -804,6 +1008,15 @@ func (n *SequenceNode) AddColumn(col int) {
}
}
// SetComment set comment token
func (n *SequenceNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
func (n *SequenceNode) flowStyleString() string {
values := []string{}
for _, value := range n.Values {
@ -849,6 +1062,7 @@ func (n *SequenceNode) ArrayRange() *ArrayNodeIter {
// AnchorNode type of anchor node
type AnchorNode struct {
Comment *token.Token // position of Comment ( `#comment` )
Start *token.Token
Name Node
Value Node
@ -862,6 +1076,11 @@ func (n *AnchorNode) GetToken() *token.Token {
return n.Start
}
// GetComment returns comment token instance
func (n *AnchorNode) GetComment() *token.Token {
return n.Comment
}
// AddColumn add column number to child nodes recursively
func (n *AnchorNode) AddColumn(col int) {
n.Start.AddColumn(col)
@ -873,6 +1092,15 @@ func (n *AnchorNode) AddColumn(col int) {
}
}
// SetComment set comment token
func (n *AnchorNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
// String anchor to text
func (n *AnchorNode) String() string {
value := n.Value.String()
@ -888,6 +1116,7 @@ func (n *AnchorNode) String() string {
// AliasNode type of alias node
type AliasNode struct {
Comment *token.Token // position of Comment ( `#comment` )
Start *token.Token
Value Node
}
@ -900,6 +1129,11 @@ func (n *AliasNode) GetToken() *token.Token {
return n.Start
}
// GetComment returns comment token instance
func (n *AliasNode) GetComment() *token.Token {
return n.Comment
}
// AddColumn add column number to child nodes recursively
func (n *AliasNode) AddColumn(col int) {
n.Start.AddColumn(col)
@ -908,6 +1142,15 @@ func (n *AliasNode) AddColumn(col int) {
}
}
// SetComment set comment token
func (n *AliasNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
// String alias to text
func (n *AliasNode) String() string {
return fmt.Sprintf("*%s", n.Value.String())
@ -915,6 +1158,7 @@ func (n *AliasNode) String() string {
// DirectiveNode type of directive node
type DirectiveNode struct {
Comment *token.Token // position of Comment ( `#comment` )
Start *token.Token
Value Node
}
@ -927,6 +1171,11 @@ func (n *DirectiveNode) GetToken() *token.Token {
return n.Start
}
// GetComment returns comment token instance
func (n *DirectiveNode) GetComment() *token.Token {
return n.Comment
}
// AddColumn add column number to child nodes recursively
func (n *DirectiveNode) AddColumn(col int) {
if n.Value != nil {
@ -934,6 +1183,15 @@ func (n *DirectiveNode) AddColumn(col int) {
}
}
// SetComment set comment token
func (n *DirectiveNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
// String directive to text
func (n *DirectiveNode) String() string {
return fmt.Sprintf("%s%s", n.Start.Value, n.Value.String())
@ -941,6 +1199,7 @@ func (n *DirectiveNode) String() string {
// TagNode type of tag node
type TagNode struct {
Comment *token.Token // position of Comment ( `#comment` )
Start *token.Token
Value Node
}
@ -953,6 +1212,11 @@ func (n *TagNode) GetToken() *token.Token {
return n.Start
}
// GetComment returns comment token instance
func (n *TagNode) GetComment() *token.Token {
return n.Comment
}
// AddColumn add column number to child nodes recursively
func (n *TagNode) AddColumn(col int) {
n.Start.AddColumn(col)
@ -961,11 +1225,53 @@ func (n *TagNode) AddColumn(col int) {
}
}
// SetComment set comment token
func (n *TagNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
// String tag to text
func (n *TagNode) String() string {
return fmt.Sprintf("%s %s", n.Start.Value, n.Value.String())
}
// CommentNode type of comment node
type CommentNode struct {
Comment *token.Token // position of Comment ( `#comment` )
}
// Type returns TagType
func (n *CommentNode) Type() NodeType { return CommentType }
// GetToken returns token instance
func (n *CommentNode) GetToken() *token.Token { return n.Comment }
// GetComment returns comment token instance
func (n *CommentNode) GetComment() *token.Token { return n.Comment }
// AddColumn add column number to child nodes recursively
func (n *CommentNode) AddColumn(col int) {
n.Comment.AddColumn(col)
}
// SetComment set comment token
func (n *CommentNode) SetComment(tk *token.Token) error {
if tk.Type != token.CommentType {
return ErrInvalidTokenType
}
n.Comment = tk
return nil
}
// String comment to text
func (n *CommentNode) String() string {
return n.Comment.Value
}
// Visitor has Visit method that is invokded for each node encountered by Walk.
// If the result visitor w is not nil, Walk visits each of the children of node with the visitor w,
// followed by a call of w.Visit(nil).

31
parser/context.go
View file

@ -28,19 +28,42 @@ func (c *context) currentToken() *token.Token {
return c.tokens[c.idx]
}
func (c *context) nextNotCommentToken() *token.Token {
for i := c.idx; i+1 < c.size; i++ {
tk := c.tokens[i+1]
if tk.Type == token.CommentType {
continue
}
return tk
}
return nil
}
func (c *context) nextToken() *token.Token {
if c.size > c.idx+1 {
if c.idx+1 >= c.size {
return nil
}
return c.tokens[c.idx+1]
}
func (c *context) afterNextNotCommentToken() *token.Token {
for i := c.idx; i+2 < c.size; i++ {
tk := c.tokens[i+2]
if tk.Type == token.CommentType {
continue
}
return tk
}
return nil
}
func (c *context) afterNextToken() *token.Token {
if c.size > c.idx+2 {
return c.tokens[c.idx+2]
}
if c.idx+2 >= c.size {
return nil
}
return c.tokens[c.idx+2]
}
func (c *context) enabledComment() bool {
return c.mode&ParseComments != 0

75
parser/parser.go
View file

@ -16,6 +16,7 @@ type parser struct{}
func (p *parser) parseMapping(ctx *context) (ast.Node, error) {
node := ast.Mapping(ctx.currentToken(), true)
ctx.progress(1) // skip MappingStart token
p.setCommentIfExists(ctx, node)
for ctx.next() {
tk := ctx.currentToken()
if tk.Type == token.MappingEndType {
@ -113,6 +114,9 @@ func (p *parser) parseMappingValue(ctx *context) (ast.Node, error) {
ctx.progress(1) // progress to mapping value token
tk := ctx.currentToken() // get mapping value token
ctx.progress(1) // progress to value token
if err := p.setCommentIfExists(ctx, key); err != nil {
return nil, errors.Wrapf(err, "failed to set comment token to node")
}
var value ast.Node
if vtk := ctx.currentToken(); vtk == nil {
value = ast.Null(token.New("null", "null", tk.Position))
@ -127,7 +131,7 @@ func (p *parser) parseMappingValue(ctx *context) (ast.Node, error) {
valueColumn := value.GetToken().Position.Column
if keyColumn == valueColumn {
if value.Type() == ast.StringType {
ntk := ctx.nextToken()
ntk := ctx.nextNotCommentToken()
if ntk == nil || (ntk.Type != token.MappingValueType && ntk.Type != token.SequenceEntryType) {
return nil, errors.ErrSyntax("could not found expected ':' token", value.GetToken())
}
@ -139,14 +143,24 @@ func (p *parser) parseMappingValue(ctx *context) (ast.Node, error) {
Value: value,
}
ntk := ctx.nextToken()
antk := ctx.afterNextToken()
if ntk != nil && ntk.Type == token.CommentType {
// next token is comment for map value
ctx.progress(1)
if err := p.setCommentIfExists(ctx, value); err != nil {
return nil, errors.Wrapf(err, "failed to set comment token to node")
}
ntk = ctx.nextToken()
}
antk := ctx.afterNextNotCommentToken()
node := &ast.MappingNode{
Start: tk,
Values: []*ast.MappingValueNode{mvnode},
}
for antk != nil && antk.Type == token.MappingValueType &&
ntk.Position.Column == key.GetToken().Position.Column {
if ctx.currentToken().Type != token.CommentType {
ctx.progress(1)
}
value, err := p.parseToken(ctx, ctx.currentToken())
if err != nil {
return nil, errors.Wrapf(err, "failed to parse mapping node")
@ -154,7 +168,13 @@ func (p *parser) parseMappingValue(ctx *context) (ast.Node, error) {
switch value.Type() {
case ast.MappingType:
c := value.(*ast.MappingNode)
for _, v := range c.Values {
comment := c.GetComment()
for idx, v := range c.Values {
if idx == 0 && comment != nil {
if err := v.SetComment(comment); err != nil {
return nil, errors.Wrapf(err, "failed to set comment token to node")
}
}
node.Values = append(node.Values, v)
}
case ast.MappingValueType:
@ -162,8 +182,8 @@ func (p *parser) parseMappingValue(ctx *context) (ast.Node, error) {
default:
return nil, xerrors.Errorf("failed to parse mapping value node node is %s", value.Type())
}
ntk = ctx.nextToken()
antk = ctx.afterNextToken()
ntk = ctx.nextNotCommentToken()
antk = ctx.afterNextNotCommentToken()
}
if len(node.Values) == 1 {
return mvnode, nil
@ -319,6 +339,24 @@ func (p *parser) parseLiteral(ctx *context) (ast.Node, error) {
return node, nil
}
func (p *parser) setCommentIfExists(ctx *context, node ast.Node) error {
tk := ctx.currentToken()
if tk == nil {
return nil
}
if tk.Type != token.CommentType {
return nil
}
if node.GetToken().Position.Line != tk.Position.Line {
return nil
}
if err := node.SetComment(tk); err != nil {
return errors.Wrapf(err, "failed to set comment token to ast.Node")
}
ctx.progress(1)
return nil
}
func (p *parser) parseDocument(ctx *context) (*ast.Document, error) {
node := &ast.Document{Start: ctx.currentToken()}
ctx.progress(1) // skip document header token
@ -334,17 +372,36 @@ func (p *parser) parseDocument(ctx *context) (*ast.Document, error) {
return node, nil
}
func (p *parser) parseComment(ctx *context) (ast.Node, error) {
tk := ctx.currentToken()
ctx.progress(1) // skip comment token
node, err := p.parseToken(ctx, ctx.currentToken())
if err != nil {
return nil, errors.Wrapf(err, "failed to parse node after comment")
}
if node == nil {
return ast.Comment(tk), nil
}
if err := node.SetComment(tk); err != nil {
return nil, errors.Wrapf(err, "failed to set comment token to node")
}
return node, nil
}
func (p *parser) parseToken(ctx *context, tk *token.Token) (ast.Node, error) {
if tk == nil {
return nil, nil
}
if tk.NextType() == token.MappingValueType {
return p.parseMappingValue(ctx)
node, err := p.parseMappingValue(ctx)
return node, err
}
if node := p.parseScalarValue(tk); node != nil {
return node, nil
}
switch tk.Type {
case token.CommentType:
return p.parseComment(ctx)
case token.DocumentHeaderType:
return p.parseDocument(ctx)
case token.MappingStartType:
@ -375,10 +432,16 @@ func (p *parser) parse(tokens token.Tokens, mode Mode) (*ast.File, error) {
if err != nil {
return nil, errors.Wrapf(err, "failed to parse")
}
tk := ctx.currentToken()
if tk != nil && tk.Type != token.CommentType {
ctx.progress(1)
}
if node == nil {
continue
}
if err := p.setCommentIfExists(ctx, node); err != nil {
return nil, errors.Wrapf(err, "failed to set comment token to node")
}
if doc, ok := node.(*ast.Document); ok {
file.Docs = append(file.Docs, doc)
} else {

28
parser/parser_test.go
View file

@ -569,6 +569,30 @@ func TestSyntaxError(t *testing.T) {
}
}
func TestComment(t *testing.T) {
yml := `
# commentA
a: #commentB
# commentC
b: c # commentD
# commentE
d: e # commentF
# commentG
f: g # commentH
# commentI
f: g # commentJ
# commentK
`
f, err := parser.ParseBytes([]byte(yml), parser.ParseComments)
if err != nil {
t.Fatalf("%+v", err)
}
var v Visitor
for _, doc := range f.Docs {
ast.Walk(&v, doc.Body)
}
}
type Visitor struct {
}
@ -576,5 +600,9 @@ func (v *Visitor) Visit(node ast.Node) ast.Visitor {
tk := node.GetToken()
tk.Prev = nil
tk.Next = nil
if comment := node.GetComment(); comment != nil {
comment.Prev = nil
comment.Next = nil
}
return v
}