Traverse Array Operator

pkg/yqlib/lib.go

@@ -65,6 +65,7 @@ var SortKeys = &OperationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Han
 
 var CollectObject = &OperationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: CollectObjectOperator}
 var TraversePath = &OperationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
+var TraverseArray = &OperationType{Type: "TRAVERSE_ARRAY", NumArgs: 2, Precedence: 50, Handler: TraverseArrayOperator}
 
 var DocumentFilter = &OperationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
 var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: SelfOperator}

pkg/yqlib/operator_array_traverse.go

@@ -0,0 +1,91 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+	"strconv"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func TraverseArrayOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+	// lhs is an expression that will yield a bunch of arrays
+	// rhs is a collect expression that will yield indexes to retreive of the arrays
+
+	lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
+	if err != nil {
+		return nil, err
+	}
+
+	rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+	if err != nil {
+		return nil, err
+	}
+
+	var indicesToTraverse = rhs.Front().Value.(*CandidateNode).Node.Content
+
+	var matchingNodeMap = list.New()
+	for el := lhs.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		if candidate.Node.Kind == yaml.SequenceNode {
+			newNodes, err := traverseArrayWithIndices(candidate, indicesToTraverse)
+			if err != nil {
+				return nil, err
+			}
+			matchingNodeMap.PushBackList(newNodes)
+		} else {
+			log.Debugf("OperatorArray Traverse skipping %v as its a %v", candidate, candidate.Node.Tag)
+		}
+	}
+
+	return matchingNodeMap, nil
+}
+
+func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*list.List, error) {
+	log.Debug("traverseArrayWithIndices")
+	var newMatches = list.New()
+
+	if len(indices) == 0 {
+		var contents = candidate.Node.Content
+		var index int64
+		for index = 0; index < int64(len(contents)); index = index + 1 {
+
+			newMatches.PushBack(&CandidateNode{
+				Document: candidate.Document,
+				Path:     candidate.CreateChildPath(index),
+				Node:     contents[index],
+			})
+		}
+		return newMatches, nil
+
+	}
+
+	for _, indexNode := range indices {
+		index, err := strconv.ParseInt(indexNode.Value, 10, 64)
+		if err != nil {
+			return nil, err
+		}
+		indexToUse := index
+		contentLength := int64(len(candidate.Node.Content))
+		for contentLength <= index {
+			candidate.Node.Content = append(candidate.Node.Content, &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"})
+			contentLength = int64(len(candidate.Node.Content))
+		}
+
+		if indexToUse < 0 {
+			indexToUse = contentLength + indexToUse
+		}
+
+		if indexToUse < 0 {
+			return nil, fmt.Errorf("Index [%v] out of range, array size is %v", index, contentLength)
+		}
+
+		newMatches.PushBack(&CandidateNode{
+			Node:     candidate.Node.Content[indexToUse],
+			Document: candidate.Document,
+			Path:     candidate.CreateChildPath(index),
+		})
+	}
+	return newMatches, nil
+}

pkg/yqlib/operator_array_traverse_test.go

@@ -0,0 +1,91 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var traverseArrayOperatorScenarios = []expressionScenario{
+	{
+		document:   `{a: [a,b,c]}`,
+		expression: `.a[0]`,
+		expected: []string{
+			"D0, P[a 0], (!!str)::a\n",
+		},
+	},
+	{
+		document:   `{a: [a,b,c]}`,
+		expression: `.a[0, 2]`,
+		expected: []string{
+			"D0, P[a 0], (!!str)::a\n",
+			"D0, P[a 2], (!!str)::c\n",
+		},
+	},
+	{
+		document:   `{a: [a,b,c]}`,
+		expression: `.a.[0]`,
+		expected: []string{
+			"D0, P[a 0], (!!str)::a\n",
+		},
+	},
+	{
+		document:   `{a: [a,b,c]}`,
+		expression: `.a[-1]`,
+		expected: []string{
+			"D0, P[a -1], (!!str)::c\n",
+		},
+	},
+	{
+		document:   `{a: [a,b,c]}`,
+		expression: `.a.[-1]`,
+		expected: []string{
+			"D0, P[a -1], (!!str)::c\n",
+		},
+	},
+	{
+		document:   `{a: [a,b,c]}`,
+		expression: `.a[-2]`,
+		expected: []string{
+			"D0, P[a -2], (!!str)::b\n",
+		},
+	},
+	{
+		document:   `{a: [a,b,c]}`,
+		expression: `.a.[-2]`,
+		expected: []string{
+			"D0, P[a -2], (!!str)::b\n",
+		},
+	},
+	{
+		document:   `{a: [a,b,c]}`,
+		expression: `.a[]`,
+		expected: []string{
+			"D0, P[a 0], (!!str)::a\n",
+			"D0, P[a 1], (!!str)::b\n",
+			"D0, P[a 2], (!!str)::c\n",
+		},
+	},
+	{
+		document:   `{a: [a,b,c]}`,
+		expression: `.a.[]`,
+		expected: []string{
+			"D0, P[a 0], (!!str)::a\n",
+			"D0, P[a 1], (!!str)::b\n",
+			"D0, P[a 2], (!!str)::c\n",
+		},
+	},
+	{
+		document:   `{a: [a,b,c]}`,
+		expression: `.a | .[]`,
+		expected: []string{
+			"D0, P[a 0], (!!str)::a\n",
+			"D0, P[a 1], (!!str)::b\n",
+			"D0, P[a 2], (!!str)::c\n",
+		},
+	},
+}
+
+func TestTraverseArrayOperatorScenarios(t *testing.T) {
+	for _, tt := range traverseArrayOperatorScenarios {
+		testScenario(t, &tt)
+	}
+}

pkg/yqlib/path_parse_test.go

@@ -19,8 +19,18 @@ var pathTests = []struct {
 	},
 	{
 		`.a[]`,
-		append(make([]interface{}, 0), "a", "SHORT_PIPE", "[]"),
+		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "]"),
-		append(make([]interface{}, 0), "a", "[]", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
+	{
+		`.a[0]`,
+		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
+		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
+	{
+		`.a.[0]`,
+		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
+		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
 	},
 	{
 		`.a.[]`,
@@ -29,8 +39,8 @@ var pathTests = []struct {
 	},
 	{
 		`.a[].c`,
-		append(make([]interface{}, 0), "a", "SHORT_PIPE", "[]", "SHORT_PIPE", "c"),
+		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "]", "SHORT_PIPE", "c"),
-		append(make([]interface{}, 0), "a", "[]", "SHORT_PIPE", "c", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "c", "SHORT_PIPE"),
 	},
 	{
 		`[3]`,

pkg/yqlib/path_tokeniser.go

@@ -22,7 +22,6 @@ const (
 	CloseCollect
 	OpenCollectObject
 	CloseCollectObject
-	SplatOrEmptyCollect
 )
 
 type Token struct {
@@ -49,8 +48,6 @@ func (t *Token) toString() string {
 		return "{"
 	} else if t.TokenType == CloseCollectObject {
 		return "}"
-	} else if t.TokenType == SplatOrEmptyCollect {
-		return "[]?"
 	} else {
 		return "NFI"
 	}
@@ -254,8 +251,6 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte(`"[^"]*"`), stringValue(true))
 
-	lexer.Add([]byte(`\[\]`), literalToken(SplatOrEmptyCollect, true))
-
 	lexer.Add([]byte(`\[`), literalToken(OpenCollect, false))
 	lexer.Add([]byte(`\]`), literalToken(CloseCollect, true))
 	lexer.Add([]byte(`\{`), literalToken(OpenCollectObject, false))
@@ -324,25 +319,6 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) {
 func (p *pathTokeniser) handleToken(tokens []*Token, index int, postProcessedTokens []*Token) (tokensAccum []*Token, skipNextToken bool) {
 	skipNextToken = false
 	token := tokens[index]
-	if token.TokenType == SplatOrEmptyCollect {
-		if index > 0 && tokens[index-1].TokenType == OperationToken &&
-			tokens[index-1].Operation.OperationType == TraversePath {
-			// must be a splat without a preceding dot , e.g. .a[]
-			// lets put a pipe in front of it, and convert it to a traverse "[]" token
-			pipeOp := &Operation{OperationType: ShortPipe, Value: "PIPE"}
-
-			postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: pipeOp})
-
-			traverseOp := &Operation{OperationType: TraversePath, Value: "[]", StringValue: "[]"}
-			token = &Token{TokenType: OperationToken, Operation: traverseOp, CheckForPostTraverse: true}
-
-		} else {
-			// gotta be a collect empty array, we need to split this into two tokens
-			// one OpenCollect, the other CloseCollect
-			postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OpenCollect})
-			token = &Token{TokenType: CloseCollect, CheckForPostTraverse: true}
-		}
-	}
 
 	if index != len(tokens)-1 && token.AssignOperation != nil &&
 		tokens[index+1].TokenType == OperationToken &&
@@ -359,5 +335,10 @@ func (p *pathTokeniser) handleToken(tokens []*Token, index int, postProcessedTok
 		op := &Operation{OperationType: ShortPipe, Value: "PIPE"}
 		postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
 	}
+	if index != len(tokens)-1 && token.CheckForPostTraverse &&
+		tokens[index+1].TokenType == OpenCollect {
+		op := &Operation{OperationType: TraverseArray}
+		postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
+	}
 	return postProcessedTokens, skipNextToken
 }

yq_test.go

@@ -1,60 +0,0 @@
-package main
-
-// import (
-// 	"fmt"
-// 	"runtime"
-// 	"testing"
-
-// 	"github.com/mikefarah/yq/v2/pkg/marshal"
-// 	"github.com/mikefarah/yq/v2/test"
-// )
-
-// func TestMultilineString(t *testing.T) {
-// 	testString := `
-// 	abcd
-// 	efg`
-// 	formattedResult, _ := marshal.NewYamlConverter().YamlToString(testString, false)
-// 	test.AssertResult(t, testString, formattedResult)
-// }
-
-// func TestNewYaml(t *testing.T) {
-// 	result, _ := newYaml([]string{"b.c", "3"})
-// 	formattedResult := fmt.Sprintf("%v", result)
-// 	test.AssertResult(t,
-// 		"[{b [{c 3}]}]",
-// 		formattedResult)
-// }
-
-// func TestNewYamlArray(t *testing.T) {
-// 	result, _ := newYaml([]string{"[0].cat", "meow"})
-// 	formattedResult := fmt.Sprintf("%v", result)
-// 	test.AssertResult(t,
-// 		"[[{cat meow}]]",
-// 		formattedResult)
-// }
-
-// func TestNewYaml_WithScript(t *testing.T) {
-// 	writeScript = "examples/instruction_sample.yaml"
-// 	expectedResult := `b:
-//   c: cat
-//   e:
-//   - name: Mike Farah`
-// 	result, _ := newYaml([]string{""})
-// 	actualResult, _ := marshal.NewYamlConverter().YamlToString(result, true)
-// 	test.AssertResult(t, expectedResult, actualResult)
-// }
-
-// func TestNewYaml_WithUnknownScript(t *testing.T) {
-// 	writeScript = "fake-unknown"
-// 	_, err := newYaml([]string{""})
-// 	if err == nil {
-// 		t.Error("Expected error due to unknown file")
-// 	}
-// 	var expectedOutput string
-// 	if runtime.GOOS == "windows" {
-// 		expectedOutput = `open fake-unknown: The system cannot find the file specified.`
-// 	} else {
-// 		expectedOutput = `open fake-unknown: no such file or directory`
-// 	}
-// 	test.AssertResult(t, expectedOutput, err.Error())
-// }