taigrr/yq
1
0
Fork 0
mirror of https://github.com/taigrr/yq synced 2025-01-18 04:53:17 -08:00
Code Issues Projects Releases Wiki Activity

Traverse Array Operator

Browse Source
This commit is contained in:
Mike Farah 2020-12-26 21:37:08 +11:00
parent ea231006ed
commit 6a13c8b78f
6 changed files with 202 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/yqlib/lib.go
View File

@ -65,6 +65,7 @@ var SortKeys = &OperationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Han
var CollectObject = &OperationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: CollectObjectOperator}
var TraversePath = &OperationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
var TraverseArray = &OperationType{Type: "TRAVERSE_ARRAY", NumArgs: 2, Precedence: 50, Handler: TraverseArrayOperator}
var DocumentFilter = &OperationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: SelfOperator}

91
pkg/yqlib/operator_array_traverse.go Normal file
View File

@ -0,0 +1,91 @@
package yqlib
import (
"container/list"
"fmt"
"strconv"
yaml "gopkg.in/yaml.v3"
)
func TraverseArrayOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
// lhs is an expression that will yield a bunch of arrays
// rhs is a collect expression that will yield indexes to retreive of the arrays
lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
if err != nil {
return nil, err
}
rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
if err != nil {
return nil, err
}
var indicesToTraverse = rhs.Front().Value.(*CandidateNode).Node.Content
var matchingNodeMap = list.New()
for el := lhs.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
if candidate.Node.Kind == yaml.SequenceNode {
newNodes, err := traverseArrayWithIndices(candidate, indicesToTraverse)
if err != nil {
return nil, err
}
matchingNodeMap.PushBackList(newNodes)
} else {
log.Debugf("OperatorArray Traverse skipping %v as its a %v", candidate, candidate.Node.Tag)
}
}
return matchingNodeMap, nil
}
func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*list.List, error) {
log.Debug("traverseArrayWithIndices")
var newMatches = list.New()
if len(indices) == 0 {
var contents = candidate.Node.Content
var index int64
for index = 0; index < int64(len(contents)); index = index + 1 {
newMatches.PushBack(&CandidateNode{
Document: candidate.Document,
Path: candidate.CreateChildPath(index),
Node: contents[index],
})
}
return newMatches, nil
}
for _, indexNode := range indices {
index, err := strconv.ParseInt(indexNode.Value, 10, 64)
if err != nil {
return nil, err
}
indexToUse := index
contentLength := int64(len(candidate.Node.Content))
for contentLength <= index {
candidate.Node.Content = append(candidate.Node.Content, &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"})
contentLength = int64(len(candidate.Node.Content))
}
if indexToUse < 0 {
indexToUse = contentLength + indexToUse
}
if indexToUse < 0 {
return nil, fmt.Errorf("Index [%v] out of range, array size is %v", index, contentLength)
}
newMatches.PushBack(&CandidateNode{
Node: candidate.Node.Content[indexToUse],
Document: candidate.Document,
Path: candidate.CreateChildPath(index),
})
}
return newMatches, nil
}

91
pkg/yqlib/operator_array_traverse_test.go Normal file
View File

@ -0,0 +1,91 @@
package yqlib
import (
"testing"
)
var traverseArrayOperatorScenarios = []expressionScenario{
{
document: `{a: [a,b,c]}`,
expression: `.a[0]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
},
},
{
document: `{a: [a,b,c]}`,
expression: `.a[0, 2]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
"D0, P[a 2], (!!str)::c\n",
},
},
{
document: `{a: [a,b,c]}`,
expression: `.a.[0]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
},
},
{
document: `{a: [a,b,c]}`,
expression: `.a[-1]`,
expected: []string{
"D0, P[a -1], (!!str)::c\n",
},
},
{
document: `{a: [a,b,c]}`,
expression: `.a.[-1]`,
expected: []string{
"D0, P[a -1], (!!str)::c\n",
},
},
{
document: `{a: [a,b,c]}`,
expression: `.a[-2]`,
expected: []string{
"D0, P[a -2], (!!str)::b\n",
},
},
{
document: `{a: [a,b,c]}`,
expression: `.a.[-2]`,
expected: []string{
"D0, P[a -2], (!!str)::b\n",
},
},
{
document: `{a: [a,b,c]}`,
expression: `.a[]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
"D0, P[a 1], (!!str)::b\n",
"D0, P[a 2], (!!str)::c\n",
},
},
{
document: `{a: [a,b,c]}`,
expression: `.a.[]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
"D0, P[a 1], (!!str)::b\n",
"D0, P[a 2], (!!str)::c\n",
},
},
{
document: `{a: [a,b,c]}`,
expression: `.a | .[]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
"D0, P[a 1], (!!str)::b\n",
"D0, P[a 2], (!!str)::c\n",
},
},
}
func TestTraverseArrayOperatorScenarios(t *testing.T) {
for _, tt := range traverseArrayOperatorScenarios {
testScenario(t, &tt)
}
}

18
pkg/yqlib/path_parse_test.go
View File

@ -19,8 +19,18 @@ var pathTests = []struct {
},
{
`.a[]`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "[]"),
append(make([]interface{}, 0), "a", "[]", "SHORT_PIPE"),
append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "]"),
append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
},
{
`.a[0]`,
append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
},
{
`.a.[0]`,
append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
},
{
`.a.[]`,
@ -29,8 +39,8 @@ var pathTests = []struct {
},
{
`.a[].c`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "[]", "SHORT_PIPE", "c"),
append(make([]interface{}, 0), "a", "[]", "SHORT_PIPE", "c", "SHORT_PIPE"),
append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "]", "SHORT_PIPE", "c"),
append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "c", "SHORT_PIPE"),
},
{
`[3]`,

29
pkg/yqlib/path_tokeniser.go
View File

@ -22,7 +22,6 @@ const (
CloseCollect
OpenCollectObject
CloseCollectObject
SplatOrEmptyCollect
)
type Token struct {
@ -49,8 +48,6 @@ func (t *Token) toString() string {
return "{"
} else if t.TokenType == CloseCollectObject {
return "}"
} else if t.TokenType == SplatOrEmptyCollect {
return "[]?"
} else {
return "NFI"
}
@ -254,8 +251,6 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`"[^"]*"`), stringValue(true))
lexer.Add([]byte(`\[\]`), literalToken(SplatOrEmptyCollect, true))
lexer.Add([]byte(`\[`), literalToken(OpenCollect, false))
lexer.Add([]byte(`\]`), literalToken(CloseCollect, true))
lexer.Add([]byte(`\{`), literalToken(OpenCollectObject, false))
@ -324,25 +319,6 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) {
func (p *pathTokeniser) handleToken(tokens []*Token, index int, postProcessedTokens []*Token) (tokensAccum []*Token, skipNextToken bool) {
skipNextToken = false
token := tokens[index]
if token.TokenType == SplatOrEmptyCollect {
if index > 0 && tokens[index-1].TokenType == OperationToken &&
tokens[index-1].Operation.OperationType == TraversePath {
// must be a splat without a preceding dot , e.g. .a[]
// lets put a pipe in front of it, and convert it to a traverse "[]" token
pipeOp := &Operation{OperationType: ShortPipe, Value: "PIPE"}
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: pipeOp})
traverseOp := &Operation{OperationType: TraversePath, Value: "[]", StringValue: "[]"}
token = &Token{TokenType: OperationToken, Operation: traverseOp, CheckForPostTraverse: true}
} else {
// gotta be a collect empty array, we need to split this into two tokens
// one OpenCollect, the other CloseCollect
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OpenCollect})
token = &Token{TokenType: CloseCollect, CheckForPostTraverse: true}
}
}
if index != len(tokens)-1 && token.AssignOperation != nil &&
tokens[index+1].TokenType == OperationToken &&
@ -359,5 +335,10 @@ func (p *pathTokeniser) handleToken(tokens []*Token, index int, postProcessedTok
op := &Operation{OperationType: ShortPipe, Value: "PIPE"}
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
}
if index != len(tokens)-1 && token.CheckForPostTraverse &&
tokens[index+1].TokenType == OpenCollect {
op := &Operation{OperationType: TraverseArray}
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
}
return postProcessedTokens, skipNextToken
}

60
yq_test.go
View File

@ -1,60 +0,0 @@
package main
// import (
// "fmt"
// "runtime"
// "testing"
// "github.com/mikefarah/yq/v2/pkg/marshal"
// "github.com/mikefarah/yq/v2/test"
// )
// func TestMultilineString(t *testing.T) {
// testString := `
// abcd
// efg`
// formattedResult, _ := marshal.NewYamlConverter().YamlToString(testString, false)
// test.AssertResult(t, testString, formattedResult)
// }
// func TestNewYaml(t *testing.T) {
// result, _ := newYaml([]string{"b.c", "3"})
// formattedResult := fmt.Sprintf("%v", result)
// test.AssertResult(t,
// "[{b [{c 3}]}]",
// formattedResult)
// }
// func TestNewYamlArray(t *testing.T) {
// result, _ := newYaml([]string{"[0].cat", "meow"})
// formattedResult := fmt.Sprintf("%v", result)
// test.AssertResult(t,
// "[[{cat meow}]]",
// formattedResult)
// }
// func TestNewYaml_WithScript(t *testing.T) {
// writeScript = "examples/instruction_sample.yaml"
// expectedResult := `b:
// c: cat
// e:
// - name: Mike Farah`
// result, _ := newYaml([]string{""})
// actualResult, _ := marshal.NewYamlConverter().YamlToString(result, true)
// test.AssertResult(t, expectedResult, actualResult)
// }
// func TestNewYaml_WithUnknownScript(t *testing.T) {
// writeScript = "fake-unknown"
// _, err := newYaml([]string{""})
// if err == nil {
// t.Error("Expected error due to unknown file")
// }
// var expectedOutput string
// if runtime.GOOS == "windows" {
// expectedOutput = `open fake-unknown: The system cannot find the file specified.`
// } else {
// expectedOutput = `open fake-unknown: no such file or directory`
// }
// test.AssertResult(t, expectedOutput, err.Error())
// }