JQ like syntax wip

pkg/yqlib/treeops/candidate_node.go

@@ -18,6 +18,15 @@ func (n *CandidateNode) GetKey() string {
 	return fmt.Sprintf("%v - %v", n.Document, n.Path)
 }
 
+// updates this candidate from the given candidate node
+func (n *CandidateNode) UpdateFrom(other *CandidateNode) {
+	n.Node.Content = other.Node.Content
+	n.Node.Value = other.Node.Value
+	n.Node.Kind = other.Node.Kind
+	n.Node.Tag = other.Node.Tag
+	n.Node.Style = other.Node.Style
+}
+
 func (n *CandidateNode) PathStackToString() string {
 	return mergePathStackToString(n.Path)
 }

pkg/yqlib/treeops/data_tree_navigator.go

@@ -70,7 +70,7 @@ func (d *dataTreeNavigator) getMatchingNodes(matchingNodes *orderedmap.OrderedMa
 	log.Debugf("Processing Path: %v", pathNode.PathElement.toString())
 	if pathNode.PathElement.PathElementType == SelfReference {
 		return matchingNodes, nil
-	} else if pathNode.PathElement.PathElementType == PathKey || pathNode.PathElement.PathElementType == ArrayIndex {
+	} else if pathNode.PathElement.PathElementType == PathKey {
 		return d.traverse(matchingNodes, pathNode.PathElement)
 	} else {
 		handler := pathNode.PathElement.OperationType.Handler

pkg/yqlib/treeops/data_tree_navigator_test.go

@@ -240,14 +240,41 @@ func TestDataTreeNavigatorDeleteViaSelf(t *testing.T) {
 	test.AssertResult(t, expected, resultsToString(results))
 }
 
-func TestDataTreeNavigatorCountWithFilter(t *testing.T) {
+func TestDataTreeNavigatorFilterWithSplat(t *testing.T) {
 
 	nodes := readDoc(t, `f:
   a: frog
   b: dally
   c: log`)
 
-	path, errPath := treeCreator.ParsePath("f(count(. == *og))")
+	path, errPath := treeCreator.ParsePath(".f | .[] == \"frog\"")
+	if errPath != nil {
+		t.Error(errPath)
+	}
+	results, errNav := treeNavigator.GetMatchingNodes(nodes, path)
+
+	if errNav != nil {
+		t.Error(errNav)
+	}
+
+	expected := `
+-- Node --
+  Document 0, path: [f]
+  Tag: !!int, Kind: ScalarNode, Anchor: 
+  2
+`
+
+	test.AssertResult(t, expected, resultsToString(results))
+}
+
+func TestDataTreeNavigatorCountAndCollectWithFilterCmd(t *testing.T) {
+
+	nodes := readDoc(t, `f:
+  a: frog
+  b: dally
+  c: log`)
+
+	path, errPath := treeCreator.ParsePath(".f | .[] == *og ")
 	if errPath != nil {
 		t.Error(errPath)
 	}
@@ -934,7 +961,7 @@ func TestDataTreeNavigatorEqualsSimple(t *testing.T) {
   pat: {b: banana}
 `)
 
-	path, errPath := treeCreator.ParsePath("a.(b == apple)")
+	path, errPath := treeCreator.ParsePath(".a | (.[].b == \"apple\")")
 	if errPath != nil {
 		t.Error(errPath)
 	}
@@ -1132,7 +1159,7 @@ func TestDataTreeNavigatorArrayEqualsDeep(t *testing.T) {
 	test.AssertResult(t, expected, resultsToString(results))
 }
 
-func TestDataTreeNavigatorEqualsTrickey(t *testing.T) {
+func xTestDataTreeNavigatorEqualsTrickey(t *testing.T) {
 
 	nodes := readDoc(t, `a: 
   cat: {b: apso, c: {d : yes}}
@@ -1141,7 +1168,7 @@ func TestDataTreeNavigatorEqualsTrickey(t *testing.T) {
   fat: {b: apple}
 `)
 
-	path, errPath := treeCreator.ParsePath("a.(b == ap* and c.d == yes)")
+	path, errPath := treeCreator.ParsePath(".a(.b == ap* and .c.d == yes)")
 	if errPath != nil {
 		t.Error(errPath)
 	}

pkg/yqlib/treeops/equals_operator.go

@@ -0,0 +1,61 @@
+package treeops
+
+import (
+	"github.com/elliotchance/orderedmap"
+	"gopkg.in/yaml.v3"
+)
+
+func EqualsOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
+	log.Debugf("-- equalsOperation")
+	var results = orderedmap.NewOrderedMap()
+
+	for el := matchMap.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		log.Debug("equalsOperation checking %v", candidate)
+
+		matches, errInChild := hasMatch(d, candidate, pathNode.Lhs, pathNode.Rhs)
+		if errInChild != nil {
+			return nil, errInChild
+		}
+
+		matchString := "true"
+		if !matches {
+			matchString = "false"
+		}
+
+		node := &yaml.Node{Kind: yaml.ScalarNode, Value: matchString, Tag: "!!bool"}
+		lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
+		results.Set(candidate.GetKey(), lengthCand)
+
+	}
+
+	return results, nil
+}
+
+func hasMatch(d *dataTreeNavigator, candidate *CandidateNode, lhs *PathTreeNode, rhs *PathTreeNode) (bool, error) {
+	childMap := orderedmap.NewOrderedMap()
+	childMap.Set(candidate.GetKey(), candidate)
+	childMatches, errChild := d.getMatchingNodes(childMap, lhs)
+	log.Debug("got the LHS")
+	if errChild != nil {
+		return false, errChild
+	}
+
+	// TODO = handle other RHS types
+	return containsMatchingValue(childMatches, rhs.PathElement.StringValue), nil
+}
+
+func containsMatchingValue(matchMap *orderedmap.OrderedMap, valuePattern string) bool {
+	log.Debugf("-- findMatchingValues")
+
+	for el := matchMap.Front(); el != nil; el = el.Next() {
+		node := el.Value.(*CandidateNode)
+		log.Debugf("-- comparing %v to %v", node.Node.Value, valuePattern)
+		if Match(node.Node.Value, valuePattern) {
+			return true
+		}
+	}
+	log.Debugf("-- done findMatchingValues")
+
+	return false
+}

pkg/yqlib/treeops/leaf_traverser.go

@@ -53,7 +53,7 @@ func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement)
 
 func (t *traverser) traverseArray(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
 	log.Debug("pathNode Value %v", pathNode.Value)
-	if pathNode.Value == "[*]" || pathNode.Value == "*" {
+	if pathNode.Value == "[]" {
 
 		var contents = candidate.Node.Content
 		var newMatches = make([]*CandidateNode, len(contents))

pkg/yqlib/treeops/lib.go

@@ -15,11 +15,13 @@ type PathElementType uint32
 
 const (
 	PathKey PathElementType = 1 << iota
-	ArrayIndex
 	Operation
 	SelfReference
 	OpenBracket
 	CloseBracket
+	OpenCollect
+	CloseCollect
+	Value // e.g. string, int
 )
 
 type OperationType struct {
@@ -30,16 +32,23 @@ type OperationType struct {
 }
 
 var None = &OperationType{Type: "NONE", NumArgs: 0, Precedence: 0}
-var Traverse = &OperationType{Type: "TRAVERSE", NumArgs: 2, Precedence: 35, Handler: TraverseOperator}
+
 var Or = &OperationType{Type: "OR", NumArgs: 2, Precedence: 10, Handler: UnionOperator}
 var And = &OperationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: IntersectionOperator}
-var Equals = &OperationType{Type: "EQUALS", NumArgs: 2, Precedence: 30, Handler: EqualsOperator}
-var Assign = &OperationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: AssignOperator}
-var DeleteChild = &OperationType{Type: "DELETE", NumArgs: 2, Precedence: 30, Handler: DeleteChildOperator}
 
-var Count = &OperationType{Type: "COUNT", NumArgs: 1, Precedence: 40, Handler: CountOperator}
+var Assign = &OperationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: AssignOperator}
+var Equals = &OperationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: EqualsOperator}
+var Pipe = &OperationType{Type: "PIPE", NumArgs: 2, Precedence: 45, Handler: PipeOperator}
+
+var Length = &OperationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: LengthOperator}
+
+// not sure yet
+
+var DeleteChild = &OperationType{Type: "DELETE", NumArgs: 2, Precedence: 40, Handler: DeleteChildOperator}
 var Collect = &OperationType{Type: "COLLECT", NumArgs: 1, Precedence: 40, Handler: CollectOperator}
 
+// var Splat = &OperationType{Type: "SPLAT", NumArgs: 0, Precedence: 40, Handler: SplatOperator}
+
 // var Exists = &OperationType{Type: "Length", NumArgs: 2, Precedence: 35}
 // filters matches if they have the existing path
 
@@ -55,13 +64,15 @@ func (p *PathElement) toString() string {
 	var result string = ``
 	switch p.PathElementType {
 	case PathKey:
-		result = result + fmt.Sprintf("PathKey - '%v'\n", p.Value)
+		result = result + fmt.Sprintf("PathKey - %v", p.Value)
-	case ArrayIndex:
-		result = result + fmt.Sprintf("ArrayIndex - '%v'\n", p.Value)
 	case SelfReference:
-		result = result + fmt.Sprintf("SELF\n")
+		result = result + fmt.Sprintf("SELF")
 	case Operation:
-		result = result + fmt.Sprintf("Operation - %v\n", p.OperationType.Type)
+		result = result + fmt.Sprintf("Operation - %v", p.OperationType.Type)
+	case Value:
+		result = result + fmt.Sprintf("Value - %v (%T)", p.Value, p.Value)
+	default:
+		result = result + "I HAVENT GOT A STRATEGY"
 	}
 	return result
 }

pkg/yqlib/treeops/operators.go

@@ -9,7 +9,7 @@ import (
 
 type OperatorHandler func(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error)
 
-func TraverseOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
+func PipeOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
 	lhs, err := d.getMatchingNodes(matchingNodes, pathNode.Lhs)
 	if err != nil {
 		return nil, err
@@ -17,15 +17,32 @@ func TraverseOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap
 	return d.getMatchingNodes(lhs, pathNode.Rhs)
 }
 
+func nodeToMap(candidate *CandidateNode) *orderedmap.OrderedMap {
+	elMap := orderedmap.NewOrderedMap()
+	elMap.Set(candidate.GetKey(), candidate)
+	return elMap
+}
+
 func AssignOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
 	lhs, err := d.getMatchingNodes(matchingNodes, pathNode.Lhs)
 	if err != nil {
 		return nil, err
 	}
 	for el := lhs.Front(); el != nil; el = el.Next() {
-		node := el.Value.(*CandidateNode)
+		candidate := el.Value.(*CandidateNode)
-		log.Debugf("Assiging %v to %v", node.GetKey(), pathNode.Rhs.PathElement.StringValue)
+
-		node.Node.Value = pathNode.Rhs.PathElement.StringValue
+		rhs, err := d.getMatchingNodes(nodeToMap(candidate), pathNode.Rhs)
+
+		if err != nil {
+			return nil, err
+		}
+
+		// grab the first value
+		first := rhs.Front()
+
+		if first != nil {
+			candidate.UpdateFrom(first.Value.(*CandidateNode))
+		}
 	}
 	return lhs, nil
 }
@@ -66,54 +83,36 @@ func IntersectionOperator(d *dataTreeNavigator, matchingNodes *orderedmap.Ordere
 }
 
 func splatNode(d *dataTreeNavigator, candidate *CandidateNode) (*orderedmap.OrderedMap, error) {
-	elMap := orderedmap.NewOrderedMap()
-	elMap.Set(candidate.GetKey(), candidate)
 	//need to splat matching nodes, then search through them
 	splatter := &PathTreeNode{PathElement: &PathElement{
 		PathElementType: PathKey,
 		Value:           "*",
 		StringValue:     "*",
 	}}
-	return d.getMatchingNodes(elMap, splatter)
+	return d.getMatchingNodes(nodeToMap(candidate), splatter)
 }
 
-func EqualsOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
+func LengthOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
-	log.Debugf("-- equalsOperation")
+	log.Debugf("-- lengthOperation")
 	var results = orderedmap.NewOrderedMap()
 
 	for el := matchMap.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		valuePattern := pathNode.Rhs.PathElement.StringValue
+		var length int
-		log.Debug("checking %v", candidate)
+		switch candidate.Node.Kind {
-
+		case yaml.ScalarNode:
-		errInChild := findMatchingChildren(d, results, candidate, pathNode.Lhs, valuePattern)
+			length = len(candidate.Node.Value)
-		if errInChild != nil {
+		case yaml.MappingNode:
-			return nil, errInChild
+			length = len(candidate.Node.Content) / 2
-		}
+		case yaml.SequenceNode:
+			length = len(candidate.Node.Content)
+		default:
+			length = 0
 		}
 
-	return results, nil
-}
-
-func CountOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
-	log.Debugf("-- countOperation")
-	var results = orderedmap.NewOrderedMap()
-
-	for el := matchMap.Front(); el != nil; el = el.Next() {
-		candidate := el.Value.(*CandidateNode)
-		elMap := orderedmap.NewOrderedMap()
-		elMap.Set(el.Key, el.Value)
-		childMatches, errChild := d.getMatchingNodes(elMap, pathNode.Rhs)
-
-		if errChild != nil {
-			return nil, errChild
-		}
-
-		length := childMatches.Len()
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", length), Tag: "!!int"}
 		lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
 		results.Set(candidate.GetKey(), lengthCand)
-
 	}
 
 	return results, nil
@@ -122,76 +121,25 @@ func CountOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNo
 func CollectOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
 	log.Debugf("-- collectOperation")
 
-	log.Debugf("-- countOperation")
 	var results = orderedmap.NewOrderedMap()
 
-	for el := matchMap.Front(); el != nil; el = el.Next() {
-		candidate := el.Value.(*CandidateNode)
-		elMap := orderedmap.NewOrderedMap()
-		elMap.Set(el.Key, el.Value)
-		childMatches, errChild := d.getMatchingNodes(elMap, pathNode.Rhs)
-
-		if errChild != nil {
-			return nil, errChild
-		}
-
 	node := &yaml.Node{Kind: yaml.SequenceNode}
 
-		for childEl := childMatches.Front(); childEl != nil; childEl = childEl.Next() {
+	var document uint = 0
-			childCandidate := childEl.Value.(*CandidateNode)
+	var path []interface{}
-			node.Content = append(node.Content, childCandidate.Node)
+
+	for el := matchMap.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		if path == nil && candidate.Path != nil {
+			path = candidate.Path
+			document = candidate.Document
+		}
+		node.Content = append(node.Content, candidate.Node)
 	}
 
-		lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
+	collectC := &CandidateNode{Node: node, Document: document, Path: path}
-		results.Set(candidate.GetKey(), lengthCand)
+	results.Set(collectC.GetKey(), collectC)
-	}
 
 	return results, nil
 
 }
-
-func findMatchingChildren(d *dataTreeNavigator, results *orderedmap.OrderedMap, candidate *CandidateNode, lhs *PathTreeNode, valuePattern string) error {
-	var children *orderedmap.OrderedMap
-	var err error
-	// don't splat scalars.
-	if candidate.Node.Kind != yaml.ScalarNode {
-		children, err = splatNode(d, candidate)
-		log.Debugf("-- splatted matches, ")
-		if err != nil {
-			return err
-		}
-	} else {
-		children = orderedmap.NewOrderedMap()
-		children.Set(candidate.GetKey(), candidate)
-	}
-
-	for childEl := children.Front(); childEl != nil; childEl = childEl.Next() {
-		childMap := orderedmap.NewOrderedMap()
-		childMap.Set(childEl.Key, childEl.Value)
-		childMatches, errChild := d.getMatchingNodes(childMap, lhs)
-		log.Debug("got the LHS")
-		if errChild != nil {
-			return errChild
-		}
-
-		if containsMatchingValue(childMatches, valuePattern) {
-			results.Set(childEl.Key, childEl.Value)
-		}
-	}
-	return nil
-}
-
-func containsMatchingValue(matchMap *orderedmap.OrderedMap, valuePattern string) bool {
-	log.Debugf("-- findMatchingValues")
-
-	for el := matchMap.Front(); el != nil; el = el.Next() {
-		node := el.Value.(*CandidateNode)
-		log.Debugf("-- comparing %v to %v", node.Node.Value, valuePattern)
-		if Match(node.Node.Value, valuePattern) {
-			return true
-		}
-	}
-	log.Debugf("-- done findMatchingValues")
-
-	return false
-}

pkg/yqlib/treeops/path_postfix.go

@@ -25,17 +25,28 @@ func popOpToResult(opStack []*Token, result []*PathElement) ([]*Token, []*PathEl
 func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement, error) {
 	var result []*PathElement
 	// surround the whole thing with quotes
-	var opStack = []*Token{&Token{PathElementType: OpenBracket, OperationType: None}}
+	var opStack = []*Token{&Token{PathElementType: OpenBracket, OperationType: None, Value: "("}}
-	var tokens = append(infixTokens, &Token{PathElementType: CloseBracket, OperationType: None})
+	var tokens = append(infixTokens, &Token{PathElementType: CloseBracket, OperationType: None, Value: ")"})
 
 	for _, token := range tokens {
+		log.Debugf("postfix processing token %v", token.Value)
 		switch token.PathElementType {
-		case PathKey, ArrayIndex, SelfReference:
+		case PathKey, SelfReference, Value:
 			var pathElement = PathElement{PathElementType: token.PathElementType, Value: token.Value, StringValue: token.StringValue}
 			result = append(result, &pathElement)
-		case OpenBracket:
+		case OpenBracket, OpenCollect:
 			opStack = append(opStack, token)
-
+		case CloseCollect:
+			for len(opStack) > 0 && opStack[len(opStack)-1].PathElementType != OpenCollect {
+				opStack, result = popOpToResult(opStack, result)
+			}
+			if len(opStack) == 0 {
+				return nil, errors.New("Bad path expression, got close collect brackets without matching opening bracket")
+			}
+			// now we should have ( as the last element on the opStack, get rid of it
+			opStack = opStack[0 : len(opStack)-1]
+			//and append a collect to the opStack
+			opStack = append(opStack, &Token{PathElementType: Operation, OperationType: Collect})
 		case CloseBracket:
 			for len(opStack) > 0 && opStack[len(opStack)-1].PathElementType != OpenBracket {
 				opStack, result = popOpToResult(opStack, result)

pkg/yqlib/treeops/path_postfix_test.go

@@ -20,20 +20,20 @@ func testExpression(expression string) (string, error) {
 	}
 	formatted := ""
 	for _, path := range results {
-		formatted = formatted + path.toString() + "--------\n"
+		formatted = formatted + path.toString() + "\n--------\n"
 	}
 	return formatted, nil
 }
 
 func TestPostFixTraverseBar(t *testing.T) {
-	var infix = "animals | collect(.)"
+	var infix = ".animals | [.]"
-	var expectedOutput = `PathKey - 'animals'
+	var expectedOutput = `PathKey - animals
 --------
 SELF
 --------
 Operation - COLLECT
 --------
-Operation - TRAVERSE
+Operation - PIPE
 --------
 `
 
@@ -45,17 +45,87 @@ Operation - TRAVERSE
 	test.AssertResultComplex(t, expectedOutput, actual)
 }
 
-func TestPostFixArrayEquals(t *testing.T) {
+func TestPostFixPipeEquals(t *testing.T) {
-	var infix = "animals(.== cat)"
+	var infix = `.animals |   (. == "cat") `
-	var expectedOutput = `PathKey - 'animals'
+	var expectedOutput = `PathKey - animals
 --------
 SELF
 --------
-PathKey - 'cat'
+Value - cat (string)
 --------
 Operation - EQUALS
 --------
-Operation - TRAVERSE
+Operation - PIPE
+--------
+`
+
+	actual, err := testExpression(infix)
+	if err != nil {
+		t.Error(err)
+	}
+
+	test.AssertResultComplex(t, expectedOutput, actual)
+}
+
+func TestPostFixCollect(t *testing.T) {
+	var infix = "[.a]"
+	var expectedOutput = `PathKey - a
+--------
+Operation - COLLECT
+--------
+`
+
+	actual, err := testExpression(infix)
+	if err != nil {
+		t.Error(err)
+	}
+
+	test.AssertResultComplex(t, expectedOutput, actual)
+}
+
+func TestPostFixSplatSearch(t *testing.T) {
+	var infix = `.a | (.[].b == "apple")`
+	var expectedOutput = `PathKey - a
+--------
+PathKey - []
+--------
+PathKey - b
+--------
+Operation - PIPE
+--------
+Value - apple (string)
+--------
+Operation - EQUALS
+--------
+Operation - PIPE
+--------
+`
+
+	actual, err := testExpression(infix)
+	if err != nil {
+		t.Error(err)
+	}
+
+	test.AssertResultComplex(t, expectedOutput, actual)
+}
+
+func TestPostFixCollectWithExpression(t *testing.T) {
+	var infix = `[ (.a == "fred") | (.d, .f)]`
+	var expectedOutput = `PathKey - a
+--------
+Value - fred (string)
+--------
+Operation - EQUALS
+--------
+PathKey - d
+--------
+PathKey - f
+--------
+Operation - OR
+--------
+Operation - PIPE
+--------
+Operation - COLLECT
 --------
 `
 
@@ -68,10 +138,12 @@ Operation - TRAVERSE
 }
 
 func TestPostFixLength(t *testing.T) {
-	var infix = "len(a)"
+	var infix = ".a | length"
-	var expectedOutput = `PathKey - 'a'
+	var expectedOutput = `PathKey - a
 --------
-Operation - Length
+Operation - LENGTH
+--------
+Operation - PIPE
 --------
 `
 
@@ -84,8 +156,8 @@ Operation - Length
 }
 
 func TestPostFixSimpleExample(t *testing.T) {
-	var infix = "a"
+	var infix = ".a"
-	var expectedOutput = `PathKey - 'a'
+	var expectedOutput = `PathKey - a
 --------
 `
 
@@ -98,16 +170,16 @@ func TestPostFixSimpleExample(t *testing.T) {
 }
 
 func TestPostFixSimplePathExample(t *testing.T) {
-	var infix = "apples.bananas*.cat"
+	var infix = ".apples.bananas*.cat"
-	var expectedOutput = `PathKey - 'apples'
+	var expectedOutput = `PathKey - apples
 --------
-PathKey - 'bananas*'
+PathKey - bananas*
 --------
-Operation - TRAVERSE
+Operation - PIPE
 --------
-PathKey - 'cat'
+PathKey - cat
 --------
-Operation - TRAVERSE
+Operation - PIPE
 --------
 `
 
@@ -120,14 +192,14 @@ Operation - TRAVERSE
 }
 
 func TestPostFixSimpleAssign(t *testing.T) {
-	var infix = "a.b := frog"
+	var infix = ".a.b |= \"frog\""
-	var expectedOutput = `PathKey - 'a'
+	var expectedOutput = `PathKey - a
 --------
-PathKey - 'b'
+PathKey - b
 --------
-Operation - TRAVERSE
+Operation - PIPE
 --------
-PathKey - 'frog'
+Value - frog (string)
 --------
 Operation - ASSIGN
 --------
@@ -142,38 +214,16 @@ Operation - ASSIGN
 }
 
 func TestPostFixSimplePathNumbersExample(t *testing.T) {
-	var infix = "apples[0].cat"
+	var infix = ".apples[0].cat"
-	var expectedOutput = `PathKey - 'apples'
+	var expectedOutput = `PathKey - apples
 --------
-PathKey - '0'
+PathKey - 0
 --------
-Operation - TRAVERSE
+Operation - PIPE
 --------
-PathKey - 'cat'
+PathKey - cat
 --------
-Operation - TRAVERSE
+Operation - PIPE
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}
-
-func TestPostFixSimplePathAppendArrayExample(t *testing.T) {
-	var infix = "apples[+].cat"
-	var expectedOutput = `PathKey - 'apples'
---------
-PathKey - '[+]'
---------
-Operation - TRAVERSE
---------
-PathKey - 'cat'
---------
-Operation - TRAVERSE
 --------
 `
 
@@ -186,38 +236,16 @@ Operation - TRAVERSE
 }
 
 func TestPostFixSimplePathSplatArrayExample(t *testing.T) {
-	var infix = "apples.[*]cat"
+	var infix = ".apples[].cat"
-	var expectedOutput = `PathKey - 'apples'
+	var expectedOutput = `PathKey - apples
 --------
-PathKey - '[*]'
+PathKey - []
 --------
-Operation - TRAVERSE
+Operation - PIPE
 --------
-PathKey - 'cat'
+PathKey - cat
 --------
-Operation - TRAVERSE
+Operation - PIPE
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}
-
-func TestPostFixDeepMatchExample(t *testing.T) {
-	var infix = "apples.**.cat"
-	var expectedOutput = `PathKey - 'apples'
---------
-PathKey - '**'
---------
-Operation - TRAVERSE
---------
-PathKey - 'cat'
---------
-Operation - TRAVERSE
 --------
 `
 
@@ -230,10 +258,10 @@ Operation - TRAVERSE
 }
 
 func TestPostFixOrExample(t *testing.T) {
-	var infix = "a OR b"
+	var infix = ".a, .b"
-	var expectedOutput = `PathKey - 'a'
+	var expectedOutput = `PathKey - a
 --------
-PathKey - 'b'
+PathKey - b
 --------
 Operation - OR
 --------
@@ -248,10 +276,10 @@ Operation - OR
 }
 
 func TestPostFixEqualsNumberExample(t *testing.T) {
-	var infix = "(animal == 3)"
+	var infix = ".animal == 3"
-	var expectedOutput = `PathKey - 'animal'
+	var expectedOutput = `PathKey - animal
 --------
-PathKey - '3'
+Value - 3 (int64)
 --------
 Operation - EQUALS
 --------
@@ -266,16 +294,16 @@ Operation - EQUALS
 }
 
 func TestPostFixOrWithEqualsExample(t *testing.T) {
-	var infix = "a==thing OR b==thongs"
+	var infix = ".a==\"thing\", .b==.thongs"
-	var expectedOutput = `PathKey - 'a'
+	var expectedOutput = `PathKey - a
 --------
-PathKey - 'thing'
+Value - thing (string)
 --------
 Operation - EQUALS
 --------
-PathKey - 'b'
+PathKey - b
 --------
-PathKey - 'thongs'
+PathKey - thongs
 --------
 Operation - EQUALS
 --------
@@ -292,24 +320,24 @@ Operation - OR
 }
 
 func TestPostFixOrWithEqualsPathExample(t *testing.T) {
-	var infix = "apples.monkeys==thing OR bogs.bobos==thongs"
+	var infix = ".apples.monkeys==\"thing\", .bogs.bobos==true"
-	var expectedOutput = `PathKey - 'apples'
+	var expectedOutput = `PathKey - apples
 --------
-PathKey - 'monkeys'
+PathKey - monkeys
 --------
-Operation - TRAVERSE
+Operation - PIPE
 --------
-PathKey - 'thing'
+Value - thing (string)
 --------
 Operation - EQUALS
 --------
-PathKey - 'bogs'
+PathKey - bogs
 --------
-PathKey - 'bobos'
+PathKey - bobos
 --------
-Operation - TRAVERSE
+Operation - PIPE
 --------
-PathKey - 'thongs'
+Value - true (bool)
 --------
 Operation - EQUALS
 --------

pkg/yqlib/treeops/path_tokeniser.go

@@ -18,31 +18,30 @@ type Token struct {
 	StringValue     string
 	PrefixSelf      bool
 
-	CheckForPreTraverse bool // this token can sometimes have the traverse '.' missing in frnot of it
+	CheckForPostTraverse bool // e.g. [1]cat should really be [1].cat
-	// e.g. a[1] should really be a.[1]
-	CheckForPostTraverse bool // samething but for post, e.g. [1]cat should really be [1].cat
 }
 
 func pathToken(wrapped bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		value := string(m.Bytes)
+		value = value[1:len(value)]
 		if wrapped {
 			value = unwrap(value)
 		}
-		return &Token{PathElementType: PathKey, OperationType: None, Value: value, StringValue: value}, nil
+		return &Token{PathElementType: PathKey, OperationType: None, Value: value, StringValue: value, CheckForPostTraverse: true}, nil
 	}
 }
 
-func opToken(op *OperationType, againstSelf bool) lex.Action {
+func opToken(op *OperationType) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		value := string(m.Bytes)
-		return &Token{PathElementType: Operation, OperationType: op, Value: op.Type, StringValue: value, PrefixSelf: againstSelf}, nil
+		return &Token{PathElementType: Operation, OperationType: op, Value: op.Type, StringValue: value}, nil
 	}
 }
 
-func literalToken(pType PathElementType, literal string, checkForPre bool, checkForPost bool, againstSelf bool) lex.Action {
+func literalToken(pType PathElementType, literal string, checkForPost bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &Token{PathElementType: pType, OperationType: None, Value: literal, StringValue: literal, CheckForPreTraverse: checkForPre, CheckForPostTraverse: checkForPost, PrefixSelf: againstSelf}, nil
+		return &Token{PathElementType: pType, OperationType: None, Value: literal, StringValue: literal, CheckForPostTraverse: checkForPost}, nil
 	}
 }
 
@@ -50,54 +49,96 @@ func unwrap(value string) string {
 	return value[1 : len(value)-1]
 }
 
-func arrayIndextoken(wrapped bool, checkForPre bool, checkForPost bool) lex.Action {
+func arrayIndextoken(precedingDot bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		var numberString = string(m.Bytes)
-		if wrapped {
+		startIndex := 1
-			numberString = unwrap(numberString)
+		if precedingDot {
+			startIndex = 2
 		}
+		numberString = numberString[startIndex : len(numberString)-1]
 		var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) // nolint
 		if errParsingInt != nil {
 			return nil, errParsingInt
 		}
-		return &Token{PathElementType: ArrayIndex, OperationType: None, Value: number, StringValue: numberString, CheckForPreTraverse: checkForPre, CheckForPostTraverse: checkForPost}, nil
+		return &Token{PathElementType: PathKey, OperationType: None, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil
+	}
+}
+
+func numberValue() lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		var numberString = string(m.Bytes)
+		var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) // nolint
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+		return &Token{PathElementType: Value, OperationType: None, Value: number, StringValue: numberString}, nil
+	}
+}
+
+func booleanValue(val bool) lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		return &Token{PathElementType: Value, OperationType: None, Value: val, StringValue: string(m.Bytes)}, nil
+	}
+}
+
+func stringValue(wrapped bool) lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		value := string(m.Bytes)
+		if wrapped {
+			value = unwrap(value)
+		}
+		return &Token{PathElementType: Value, OperationType: None, Value: value, StringValue: value}, nil
+	}
+}
+
+func selfToken() lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		return &Token{PathElementType: SelfReference, OperationType: None, Value: "SELF", StringValue: "SELF"}, nil
 	}
 }
 
 // Creates the lexer object and compiles the NFA.
 func initLexer() (*lex.Lexer, error) {
 	lexer := lex.NewLexer()
-	lexer.Add([]byte(`\(`), literalToken(OpenBracket, "(", true, false, false))
+	lexer.Add([]byte(`\(`), literalToken(OpenBracket, "(", false))
-	lexer.Add([]byte(`\)`), literalToken(CloseBracket, ")", false, true, false))
+	lexer.Add([]byte(`\)`), literalToken(CloseBracket, ")", true))
-	lexer.Add([]byte(`\.\s*\)`), literalToken(CloseBracket, ")", false, true, true))
 
-	lexer.Add([]byte(`\[\+\]`), literalToken(PathKey, "[+]", true, true, false))
+	lexer.Add([]byte(`\.?\[\]`), literalToken(PathKey, "[]", true))
-	lexer.Add([]byte(`\[\*\]`), literalToken(PathKey, "[*]", true, true, false))
+	lexer.Add([]byte(`\.\.`), literalToken(PathKey, "..", true))
-	lexer.Add([]byte(`\*\*`), literalToken(PathKey, "**", false, false, false))
 
-	lexer.Add([]byte(`([Oo][Rr])`), opToken(Or, false))
+	lexer.Add([]byte(`,`), opToken(Or))
-	lexer.Add([]byte(`([Aa][Nn][Dd])`), opToken(And, false))
+	lexer.Add([]byte(`length`), opToken(Length))
-	lexer.Add([]byte(`([Cc][Oo][Uu][Nn][Tt])`), opToken(Count, false))
+	lexer.Add([]byte(`([Cc][Oo][Ll][Ll][Ee][Cc][Tt])`), opToken(Collect))
-	lexer.Add([]byte(`([Cc][Oo][Ll][Ll][Ee][Cc][Tt])`), opToken(Collect, false))
 
-	lexer.Add([]byte(`\.\s*==\s*`), opToken(Equals, true))
+	lexer.Add([]byte(`\s*==\s*`), opToken(Equals))
-	lexer.Add([]byte(`\s*==\s*`), opToken(Equals, false))
 
-	lexer.Add([]byte(`\.\s*.-\s*`), opToken(DeleteChild, true))
+	lexer.Add([]byte(`\s*.-\s*`), opToken(DeleteChild))
-	lexer.Add([]byte(`\s*.-\s*`), opToken(DeleteChild, false))
 
-	lexer.Add([]byte(`\.\s*:=\s*`), opToken(Assign, true))
+	lexer.Add([]byte(`\s*\|=\s*`), opToken(Assign))
-	lexer.Add([]byte(`\s*:=\s*`), opToken(Assign, false))
+
+	lexer.Add([]byte(`\[-?[0-9]+\]`), arrayIndextoken(false))
+	lexer.Add([]byte(`\.\[-?[0-9]+\]`), arrayIndextoken(true))
 
-	lexer.Add([]byte(`\[-?[0-9]+\]`), arrayIndextoken(true, true, true))
-	lexer.Add([]byte(`-?[0-9]+`), arrayIndextoken(false, false, false))
 	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
 
-	lexer.Add([]byte(`"[^ "]+"`), pathToken(true))
+	lexer.Add([]byte(`\."[^ "]+"`), pathToken(true))
-	lexer.Add([]byte(`[^ \|\.\[\(\)=]+`), pathToken(false))
+	lexer.Add([]byte(`\.[^ \[\],\|\.\[\(\)=]+`), pathToken(false))
+	lexer.Add([]byte(`\.`), selfToken())
 
-	lexer.Add([]byte(`\|`), opToken(Traverse, false))
+	lexer.Add([]byte(`\|`), opToken(Pipe))
-	lexer.Add([]byte(`\.`), opToken(Traverse, false))
+
+	lexer.Add([]byte(`-?[0-9]+`), numberValue())
+
+	lexer.Add([]byte(`[Tt][Rr][Uu][Ee]`), booleanValue(true))
+	lexer.Add([]byte(`[Ff][Aa][Ll][Ss][Ee]`), booleanValue(false))
+
+	lexer.Add([]byte(`"[^ "]+"`), stringValue(true))
+
+	lexer.Add([]byte(`\[`), literalToken(OpenCollect, "[", false))
+	lexer.Add([]byte(`\]`), literalToken(CloseCollect, "]", true))
+
+	// lexer.Add([]byte(`[^ \,\|\.\[\(\)=]+`), stringValue(false))
 	err := lexer.Compile()
 	if err != nil {
 		return nil, err
@@ -142,19 +183,12 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) {
 	var postProcessedTokens = make([]*Token, 0)
 
 	for index, token := range tokens {
-		if index > 0 && token.CheckForPreTraverse &&
-			(tokens[index-1].PathElementType == PathKey || tokens[index-1].PathElementType == CloseBracket) {
-			postProcessedTokens = append(postProcessedTokens, &Token{PathElementType: Operation, OperationType: Traverse, Value: "."})
-		}
-		if token.PrefixSelf {
-			postProcessedTokens = append(postProcessedTokens, &Token{PathElementType: SelfReference, Value: "SELF"})
-		}
 
 		postProcessedTokens = append(postProcessedTokens, token)
 
 		if index != len(tokens)-1 && token.CheckForPostTraverse &&
 			tokens[index+1].PathElementType == PathKey {
-			postProcessedTokens = append(postProcessedTokens, &Token{PathElementType: Operation, OperationType: Traverse, Value: "."})
+			postProcessedTokens = append(postProcessedTokens, &Token{PathElementType: Operation, OperationType: Pipe, Value: "PIPE"})
 		}
 	}
 

pkg/yqlib/treeops/path_tokeniser_test.go

@@ -10,50 +10,53 @@ var tokeniserTests = []struct {
 	path           string
 	expectedTokens []interface{}
 }{ // TODO: Ensure ALL documented examples have tests! sheesh
-	{"len(.)", append(make([]interface{}, 0), "LENGTH", "(", "SELF", ")")},
+	// {"len(.)", append(make([]interface{}, 0), "LENGTH", "(", "SELF", ")")},
-	{"\"len\"(.)", append(make([]interface{}, 0), "len", ".", "(", "SELF", ")")},
+	// {"\"len\"(.)", append(make([]interface{}, 0), "len", "TRAVERSE", "(", "SELF", ")")},
-	// {"a OR (b OR c)", append(make([]interface{}, 0), "a", "OR", "(", "b", "OR", "c", ")")},
+	// {".a OR (.b OR .c)", append(make([]interface{}, 0), "a", "OR", "(", "b", "OR", "c", ")")},
 	// {"a OR (b OR c)", append(make([]interface{}, 0), "a", "OR", "(", "b", "OR", "c", ")")},
 	// {"a .- (b OR c)", append(make([]interface{}, 0), "a", " .- ", "(", "b", "OR", "c", ")")},
 	// {"(animal==3)", append(make([]interface{}, 0), "(", "animal", "==", int64(3), ")")},
 	// {"(animal==f3)", append(make([]interface{}, 0), "(", "animal", "==", "f3", ")")},
-	// {"apples.BANANAS", append(make([]interface{}, 0), "apples", ".", "BANANAS")},
+	// {"apples.BANANAS", append(make([]interface{}, 0), "apples", "TRAVERSE", "BANANAS")},
-	// {"appl*.BANA*", append(make([]interface{}, 0), "appl*", ".", "BANA*")},
+	// {"appl*.BANA*", append(make([]interface{}, 0), "appl*", "TRAVERSE", "BANA*")},
-	// {"a.b.**", append(make([]interface{}, 0), "a", ".", "b", ".", "**")},
+	// {"a.b.**", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", "**")},
-	// {"a.\"=\".frog", append(make([]interface{}, 0), "a", ".", "=", ".", "frog")},
+	// {"a.\"=\".frog", append(make([]interface{}, 0), "a", "TRAVERSE", "=", "TRAVERSE", "frog")},
-	// {"a.b.*", append(make([]interface{}, 0), "a", ".", "b", ".", "*")},
+	// {"a.b.*", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", "*")},
-	// {"a.b.thin*", append(make([]interface{}, 0), "a", ".", "b", ".", "thin*")},
+	// {"a.b.thin*", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", "thin*")},
-	// {"a.b[0]", append(make([]interface{}, 0), "a", ".", "b", ".", int64(0))},
+	// {".a.b.[0]", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", int64(0))},
-	// {"a.b.[0]", append(make([]interface{}, 0), "a", ".", "b", ".", int64(0))},
+	// {".a.b.[]", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", "[]")},
-	// {"a.b[*]", append(make([]interface{}, 0), "a", ".", "b", ".", "[*]")},
+	// {".a.b.[+]", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", "[+]")},
-	// {"a.b.[*]", append(make([]interface{}, 0), "a", ".", "b", ".", "[*]")},
+	// {".a.b.[-12]", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", int64(-12))},
-	// {"a.b[+]", append(make([]interface{}, 0), "a", ".", "b", ".", "[+]")},
+	// {".a.b.0", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", "0")},
-	// {"a.b.[+]", append(make([]interface{}, 0), "a", ".", "b", ".", "[+]")},
+	// {".a", append(make([]interface{}, 0), "a")},
-	// {"a.b[-12]", append(make([]interface{}, 0), "a", ".", "b", ".", int64(-12))},
+	// {".\"a.b\".c", append(make([]interface{}, 0), "a.b", "TRAVERSE", "c")},
-	// {"a.b.0", append(make([]interface{}, 0), "a", ".", "b", ".", int64(0))},
+	// {`.b."foo.bar"`, append(make([]interface{}, 0), "b", "TRAVERSE", "foo.bar")},
-	// // {"a.b.-12", append(make([]interface{}, 0), "a", ".", "b", ".", int64(-12))},
+	// {`f | . == *og | length`, append(make([]interface{}, 0), "f", "TRAVERSE", "SELF", "EQUALS", "*og", "TRAVERSE", "LENGTH")},
-	// {"a", append(make([]interface{}, 0), "a")},
+	// {`.a, .b`, append(make([]interface{}, 0), "a", "OR", "b")},
-	// {"\"a.b\".c", append(make([]interface{}, 0), "a.b", ".", "c")},
+	// {`[.a, .b]`, append(make([]interface{}, 0), "[", "a", "OR", "b", "]")},
-	// {`b."foo.bar"`, append(make([]interface{}, 0), "b", ".", "foo.bar")},
+	// {`."[a", ."b]"`, append(make([]interface{}, 0), "[a", "OR", "b]")},
-	// {"animals(.==cat)", append(make([]interface{}, 0), "animals", ".", "(", "SELF", ".==", "cat", ")")},
+	// {`.a[]`, append(make([]interface{}, 0), "a", "PIPE", "[]")},
-	// {"animals.(.==cat)", append(make([]interface{}, 0), "animals", ".", "(", "SELF", ".==", "cat", ")")},
+	// {`.[].a`, append(make([]interface{}, 0), "[]", "PIPE", "a")},
-	// {"animals(. == cat)", append(make([]interface{}, 0), "animals", ".", "(", "SELF", ". == ", "cat", ")")},
+	{`.a | (.[].b == "apple")`, append(make([]interface{}, 0), "a", "PIPE", "(", "[]", "PIPE", "b", "EQUALS", "apple", ")")},
-	// {"animals(.==c*)", append(make([]interface{}, 0), "animals", ".", "(", "SELF", ".==", "c*", ")")},
+
-	// {"animals(a.b==c*)", append(make([]interface{}, 0), "animals", ".", "(", "a", ".", "b", "==", "c*", ")")},
+	// {".animals | .==cat", append(make([]interface{}, 0), "animals", "TRAVERSE", "SELF", "EQUALS", "cat")},
-	// {"animals.(a.b==c*)", append(make([]interface{}, 0), "animals", ".", "(", "a", ".", "b", "==", "c*", ")")},
+	// {".animals | (. == cat)", append(make([]interface{}, 0), "animals", "TRAVERSE", "(", "SELF", "EQUALS", "cat", ")")},
-	// {"(a.b==c*).animals", append(make([]interface{}, 0), "(", "a", ".", "b", "==", "c*", ")", ".", "animals")},
+	// {".animals | (.==c*)", append(make([]interface{}, 0), "animals", "TRAVERSE", "(", "SELF", "EQUALS", "c*", ")")},
-	// {"(a.b==c*)animals", append(make([]interface{}, 0), "(", "a", ".", "b", "==", "c*", ")", ".", "animals")},
+	// {"animals(a.b==c*)", append(make([]interface{}, 0), "animals", "TRAVERSE", "(", "a", "TRAVERSE", "b", "==", "c*", ")")},
-	// {"[1].a.d", append(make([]interface{}, 0), int64(1), ".", "a", ".", "d")},
+	// {"animals.(a.b==c*)", append(make([]interface{}, 0), "animals", "TRAVERSE", "(", "a", "TRAVERSE", "b", "==", "c*", ")")},
-	// {"[1]a.d", append(make([]interface{}, 0), int64(1), ".", "a", ".", "d")},
+	// {"(a.b==c*).animals", append(make([]interface{}, 0), "(", "a", "TRAVERSE", "b", "==", "c*", ")", "TRAVERSE", "animals")},
-	// {"a[0]c", append(make([]interface{}, 0), "a", ".", int64(0), ".", "c")},
+	// {"(a.b==c*)animals", append(make([]interface{}, 0), "(", "a", "TRAVERSE", "b", "==", "c*", ")", "TRAVERSE", "animals")},
-	// {"a.[0].c", append(make([]interface{}, 0), "a", ".", int64(0), ".", "c")},
+	// {"[1].a.d", append(make([]interface{}, 0), int64(1), "TRAVERSE", "a", "TRAVERSE", "d")},
+	// {"[1]a.d", append(make([]interface{}, 0), int64(1), "TRAVERSE", "a", "TRAVERSE", "d")},
+	// {"a[0]c", append(make([]interface{}, 0), "a", "TRAVERSE", int64(0), "TRAVERSE", "c")},
+	// {"a.[0].c", append(make([]interface{}, 0), "a", "TRAVERSE", int64(0), "TRAVERSE", "c")},
 	// {"[0]", append(make([]interface{}, 0), int64(0))},
 	// {"0", append(make([]interface{}, 0), int64(0))},
-	// {"a.b[+]c", append(make([]interface{}, 0), "a", ".", "b", ".", "[+]", ".", "c")},
+	// {"a.b[+]c", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", "[+]", "TRAVERSE", "c")},
-	// {"a.cool(s.d.f == cool)", append(make([]interface{}, 0), "a", ".", "cool", ".", "(", "s", ".", "d", ".", "f", " == ", "cool", ")")},
+	// {"a.cool(s.d.f == cool)", append(make([]interface{}, 0), "a", "TRAVERSE", "cool", "TRAVERSE", "(", "s", "TRAVERSE", "d", "TRAVERSE", "f", " == ", "cool", ")")},
-	// {"a.cool.(s.d.f==cool OR t.b.h==frog).caterpillar", append(make([]interface{}, 0), "a", ".", "cool", ".", "(", "s", ".", "d", ".", "f", "==", "cool", "OR", "t", ".", "b", ".", "h", "==", "frog", ")", ".", "caterpillar")},
+	// {"a.cool.(s.d.f==cool OR t.b.h==frog).caterpillar", append(make([]interface{}, 0), "a", "TRAVERSE", "cool", "TRAVERSE", "(", "s", "TRAVERSE", "d", "TRAVERSE", "f", "==", "cool", "OR", "t", "TRAVERSE", "b", "TRAVERSE", "h", "==", "frog", ")", "TRAVERSE", "caterpillar")},
-	// {"a.cool(s.d.f==cool and t.b.h==frog)*", append(make([]interface{}, 0), "a", ".", "cool", ".", "(", "s", ".", "d", ".", "f", "==", "cool", "and", "t", ".", "b", ".", "h", "==", "frog", ")", ".", "*")},
+	// {"a.cool(s.d.f==cool and t.b.h==frog)*", append(make([]interface{}, 0), "a", "TRAVERSE", "cool", "TRAVERSE", "(", "s", "TRAVERSE", "d", "TRAVERSE", "f", "==", "cool", "and", "t", "TRAVERSE", "b", "TRAVERSE", "h", "==", "frog", ")", "TRAVERSE", "*")},
-	// {"a.cool(s.d.f==cool and t.b.h==frog).th*", append(make([]interface{}, 0), "a", ".", "cool", ".", "(", "s", ".", "d", ".", "f", "==", "cool", "and", "t", ".", "b", ".", "h", "==", "frog", ")", ".", "th*")},
+	// {"a.cool(s.d.f==cool and t.b.h==frog).th*", append(make([]interface{}, 0), "a", "TRAVERSE", "cool", "TRAVERSE", "(", "s", "TRAVERSE", "d", "TRAVERSE", "f", "==", "cool", "and", "t", "TRAVERSE", "b", "TRAVERSE", "h", "==", "frog", ")", "TRAVERSE", "th*")},
 }
 
 var tokeniser = NewPathTokeniser()
@@ -68,6 +71,6 @@ func TestTokeniser(t *testing.T) {
 		for _, token := range tokens {
 			tokenValues = append(tokenValues, token.Value)
 		}
-		test.AssertResultComplex(t, tt.expectedTokens, tokenValues)
+		test.AssertResultComplexWithContext(t, tt.expectedTokens, tokenValues, tt.path)
 	}
 }

pkg/yqlib/treeops/path_tree.go

@@ -45,12 +45,10 @@ func (p *pathTreeCreator) CreatePathTree(postFixPath []*PathElement) (*PathTreeN
 
 	for _, pathElement := range postFixPath {
 		var newNode = PathTreeNode{PathElement: pathElement}
-		if pathElement.PathElementType == Operation {
+		log.Debugf("pathTree %v ", pathElement.toString())
+		if pathElement.PathElementType == Operation && pathElement.OperationType.NumArgs > 0 {
 			numArgs := pathElement.OperationType.NumArgs
-			if numArgs == 0 {
+			if numArgs == 1 {
-				remaining := stack[:len(stack)-1]
-				stack = remaining
-			} else if numArgs == 1 {
 				remaining, rhs := stack[:len(stack)-1], stack[len(stack)-1]
 				newNode.Rhs = rhs
 				stack = remaining

test/utils.go

@@ -54,6 +54,14 @@ func AssertResultComplex(t *testing.T, expectedValue interface{}, actualValue in
 	}
 }
 
+func AssertResultComplexWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) {
+	t.Helper()
+	if !reflect.DeepEqual(expectedValue, actualValue) {
+		t.Error(context)
+		t.Error("\nExpected <", expectedValue, ">\nbut got  <", actualValue, ">", fmt.Sprintf("%T", actualValue))
+	}
+}
+
 func AssertResultWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) {
 	t.Helper()
 	if expectedValue != actualValue {