From 6a13c8b78ff385a0df7a813307e05bdea0b48b00 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 26 Dec 2020 21:37:08 +1100 Subject: [PATCH] Traverse Array Operator --- pkg/yqlib/lib.go | 1 + pkg/yqlib/operator_array_traverse.go | 91 +++++++++++++++++++++++ pkg/yqlib/operator_array_traverse_test.go | 91 +++++++++++++++++++++++ pkg/yqlib/path_parse_test.go | 18 ++++- pkg/yqlib/path_tokeniser.go | 29 ++------ yq_test.go | 60 --------------- 6 files changed, 202 insertions(+), 88 deletions(-) create mode 100644 pkg/yqlib/operator_array_traverse.go create mode 100644 pkg/yqlib/operator_array_traverse_test.go delete mode 100644 yq_test.go diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 2786d82..7d5b54a 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -65,6 +65,7 @@ var SortKeys = &OperationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Han var CollectObject = &OperationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: CollectObjectOperator} var TraversePath = &OperationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator} +var TraverseArray = &OperationType{Type: "TRAVERSE_ARRAY", NumArgs: 2, Precedence: 50, Handler: TraverseArrayOperator} var DocumentFilter = &OperationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator} var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: SelfOperator} diff --git a/pkg/yqlib/operator_array_traverse.go b/pkg/yqlib/operator_array_traverse.go new file mode 100644 index 0000000..125d0ab --- /dev/null +++ b/pkg/yqlib/operator_array_traverse.go @@ -0,0 +1,91 @@ +package yqlib + +import ( + "container/list" + "fmt" + "strconv" + + yaml "gopkg.in/yaml.v3" +) + +func TraverseArrayOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { + // lhs is an expression that will yield a bunch of arrays + // rhs is a collect expression that will yield indexes to retreive of the arrays + + lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs) + if err != nil { + return nil, err + } + + rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs) + if err != nil { + return nil, err + } + + var indicesToTraverse = rhs.Front().Value.(*CandidateNode).Node.Content + + var matchingNodeMap = list.New() + for el := lhs.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + + if candidate.Node.Kind == yaml.SequenceNode { + newNodes, err := traverseArrayWithIndices(candidate, indicesToTraverse) + if err != nil { + return nil, err + } + matchingNodeMap.PushBackList(newNodes) + } else { + log.Debugf("OperatorArray Traverse skipping %v as its a %v", candidate, candidate.Node.Tag) + } + } + + return matchingNodeMap, nil +} + +func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*list.List, error) { + log.Debug("traverseArrayWithIndices") + var newMatches = list.New() + + if len(indices) == 0 { + var contents = candidate.Node.Content + var index int64 + for index = 0; index < int64(len(contents)); index = index + 1 { + + newMatches.PushBack(&CandidateNode{ + Document: candidate.Document, + Path: candidate.CreateChildPath(index), + Node: contents[index], + }) + } + return newMatches, nil + + } + + for _, indexNode := range indices { + index, err := strconv.ParseInt(indexNode.Value, 10, 64) + if err != nil { + return nil, err + } + indexToUse := index + contentLength := int64(len(candidate.Node.Content)) + for contentLength <= index { + candidate.Node.Content = append(candidate.Node.Content, &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}) + contentLength = int64(len(candidate.Node.Content)) + } + + if indexToUse < 0 { + indexToUse = contentLength + indexToUse + } + + if indexToUse < 0 { + return nil, fmt.Errorf("Index [%v] out of range, array size is %v", index, contentLength) + } + + newMatches.PushBack(&CandidateNode{ + Node: candidate.Node.Content[indexToUse], + Document: candidate.Document, + Path: candidate.CreateChildPath(index), + }) + } + return newMatches, nil +} diff --git a/pkg/yqlib/operator_array_traverse_test.go b/pkg/yqlib/operator_array_traverse_test.go new file mode 100644 index 0000000..0a7432c --- /dev/null +++ b/pkg/yqlib/operator_array_traverse_test.go @@ -0,0 +1,91 @@ +package yqlib + +import ( + "testing" +) + +var traverseArrayOperatorScenarios = []expressionScenario{ + { + document: `{a: [a,b,c]}`, + expression: `.a[0]`, + expected: []string{ + "D0, P[a 0], (!!str)::a\n", + }, + }, + { + document: `{a: [a,b,c]}`, + expression: `.a[0, 2]`, + expected: []string{ + "D0, P[a 0], (!!str)::a\n", + "D0, P[a 2], (!!str)::c\n", + }, + }, + { + document: `{a: [a,b,c]}`, + expression: `.a.[0]`, + expected: []string{ + "D0, P[a 0], (!!str)::a\n", + }, + }, + { + document: `{a: [a,b,c]}`, + expression: `.a[-1]`, + expected: []string{ + "D0, P[a -1], (!!str)::c\n", + }, + }, + { + document: `{a: [a,b,c]}`, + expression: `.a.[-1]`, + expected: []string{ + "D0, P[a -1], (!!str)::c\n", + }, + }, + { + document: `{a: [a,b,c]}`, + expression: `.a[-2]`, + expected: []string{ + "D0, P[a -2], (!!str)::b\n", + }, + }, + { + document: `{a: [a,b,c]}`, + expression: `.a.[-2]`, + expected: []string{ + "D0, P[a -2], (!!str)::b\n", + }, + }, + { + document: `{a: [a,b,c]}`, + expression: `.a[]`, + expected: []string{ + "D0, P[a 0], (!!str)::a\n", + "D0, P[a 1], (!!str)::b\n", + "D0, P[a 2], (!!str)::c\n", + }, + }, + { + document: `{a: [a,b,c]}`, + expression: `.a.[]`, + expected: []string{ + "D0, P[a 0], (!!str)::a\n", + "D0, P[a 1], (!!str)::b\n", + "D0, P[a 2], (!!str)::c\n", + }, + }, + { + document: `{a: [a,b,c]}`, + expression: `.a | .[]`, + expected: []string{ + "D0, P[a 0], (!!str)::a\n", + "D0, P[a 1], (!!str)::b\n", + "D0, P[a 2], (!!str)::c\n", + }, + }, +} + +func TestTraverseArrayOperatorScenarios(t *testing.T) { + for _, tt := range traverseArrayOperatorScenarios { + testScenario(t, &tt) + } +} diff --git a/pkg/yqlib/path_parse_test.go b/pkg/yqlib/path_parse_test.go index 1767311..1dfa68d 100644 --- a/pkg/yqlib/path_parse_test.go +++ b/pkg/yqlib/path_parse_test.go @@ -19,8 +19,18 @@ var pathTests = []struct { }, { `.a[]`, - append(make([]interface{}, 0), "a", "SHORT_PIPE", "[]"), - append(make([]interface{}, 0), "a", "[]", "SHORT_PIPE"), + append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "]"), + append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"), + }, + { + `.a[0]`, + append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"), + append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"), + }, + { + `.a.[0]`, + append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"), + append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"), }, { `.a.[]`, @@ -29,8 +39,8 @@ var pathTests = []struct { }, { `.a[].c`, - append(make([]interface{}, 0), "a", "SHORT_PIPE", "[]", "SHORT_PIPE", "c"), - append(make([]interface{}, 0), "a", "[]", "SHORT_PIPE", "c", "SHORT_PIPE"), + append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "]", "SHORT_PIPE", "c"), + append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "c", "SHORT_PIPE"), }, { `[3]`, diff --git a/pkg/yqlib/path_tokeniser.go b/pkg/yqlib/path_tokeniser.go index 6e55173..0cd68f0 100644 --- a/pkg/yqlib/path_tokeniser.go +++ b/pkg/yqlib/path_tokeniser.go @@ -22,7 +22,6 @@ const ( CloseCollect OpenCollectObject CloseCollectObject - SplatOrEmptyCollect ) type Token struct { @@ -49,8 +48,6 @@ func (t *Token) toString() string { return "{" } else if t.TokenType == CloseCollectObject { return "}" - } else if t.TokenType == SplatOrEmptyCollect { - return "[]?" } else { return "NFI" } @@ -254,8 +251,6 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`"[^"]*"`), stringValue(true)) - lexer.Add([]byte(`\[\]`), literalToken(SplatOrEmptyCollect, true)) - lexer.Add([]byte(`\[`), literalToken(OpenCollect, false)) lexer.Add([]byte(`\]`), literalToken(CloseCollect, true)) lexer.Add([]byte(`\{`), literalToken(OpenCollectObject, false)) @@ -324,25 +319,6 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) { func (p *pathTokeniser) handleToken(tokens []*Token, index int, postProcessedTokens []*Token) (tokensAccum []*Token, skipNextToken bool) { skipNextToken = false token := tokens[index] - if token.TokenType == SplatOrEmptyCollect { - if index > 0 && tokens[index-1].TokenType == OperationToken && - tokens[index-1].Operation.OperationType == TraversePath { - // must be a splat without a preceding dot , e.g. .a[] - // lets put a pipe in front of it, and convert it to a traverse "[]" token - pipeOp := &Operation{OperationType: ShortPipe, Value: "PIPE"} - - postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: pipeOp}) - - traverseOp := &Operation{OperationType: TraversePath, Value: "[]", StringValue: "[]"} - token = &Token{TokenType: OperationToken, Operation: traverseOp, CheckForPostTraverse: true} - - } else { - // gotta be a collect empty array, we need to split this into two tokens - // one OpenCollect, the other CloseCollect - postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OpenCollect}) - token = &Token{TokenType: CloseCollect, CheckForPostTraverse: true} - } - } if index != len(tokens)-1 && token.AssignOperation != nil && tokens[index+1].TokenType == OperationToken && @@ -359,5 +335,10 @@ func (p *pathTokeniser) handleToken(tokens []*Token, index int, postProcessedTok op := &Operation{OperationType: ShortPipe, Value: "PIPE"} postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op}) } + if index != len(tokens)-1 && token.CheckForPostTraverse && + tokens[index+1].TokenType == OpenCollect { + op := &Operation{OperationType: TraverseArray} + postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op}) + } return postProcessedTokens, skipNextToken } diff --git a/yq_test.go b/yq_test.go deleted file mode 100644 index b131d01..0000000 --- a/yq_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package main - -// import ( -// "fmt" -// "runtime" -// "testing" - -// "github.com/mikefarah/yq/v2/pkg/marshal" -// "github.com/mikefarah/yq/v2/test" -// ) - -// func TestMultilineString(t *testing.T) { -// testString := ` -// abcd -// efg` -// formattedResult, _ := marshal.NewYamlConverter().YamlToString(testString, false) -// test.AssertResult(t, testString, formattedResult) -// } - -// func TestNewYaml(t *testing.T) { -// result, _ := newYaml([]string{"b.c", "3"}) -// formattedResult := fmt.Sprintf("%v", result) -// test.AssertResult(t, -// "[{b [{c 3}]}]", -// formattedResult) -// } - -// func TestNewYamlArray(t *testing.T) { -// result, _ := newYaml([]string{"[0].cat", "meow"}) -// formattedResult := fmt.Sprintf("%v", result) -// test.AssertResult(t, -// "[[{cat meow}]]", -// formattedResult) -// } - -// func TestNewYaml_WithScript(t *testing.T) { -// writeScript = "examples/instruction_sample.yaml" -// expectedResult := `b: -// c: cat -// e: -// - name: Mike Farah` -// result, _ := newYaml([]string{""}) -// actualResult, _ := marshal.NewYamlConverter().YamlToString(result, true) -// test.AssertResult(t, expectedResult, actualResult) -// } - -// func TestNewYaml_WithUnknownScript(t *testing.T) { -// writeScript = "fake-unknown" -// _, err := newYaml([]string{""}) -// if err == nil { -// t.Error("Expected error due to unknown file") -// } -// var expectedOutput string -// if runtime.GOOS == "windows" { -// expectedOutput = `open fake-unknown: The system cannot find the file specified.` -// } else { -// expectedOutput = `open fake-unknown: no such file or directory` -// } -// test.AssertResult(t, expectedOutput, err.Error()) -// }