Incrementing version

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.3.1"
+	Version = "4.3.2"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.3.1
+FROM mikefarah/yq:4.3.2
 
 COPY entrypoint.sh /entrypoint.sh
 

go.sum

@@ -281,6 +281,7 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=

pkg/yqlib/all_at_once_evaluator.go

@@ -1,29 +1,50 @@
 package yqlib
 
-import "container/list"
+import (
+	"container/list"
 
-/**
-	Loads all yaml documents of all files given into memory, then runs the given expression once.
-**/
+	yaml "gopkg.in/yaml.v3"
+)
+
+// A yaml expression evaluator that runs the expression once against all files/nodes in memory.
 type Evaluator interface {
 	EvaluateFiles(expression string, filenames []string, printer Printer) error
+
+	// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
+	EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)
+
+	// EvaluateCandidateNodes takes an expression and list of candidate nodes, returning a list of matching candidate nodes
+	EvaluateCandidateNodes(expression string, inputCandidateNodes *list.List) (*list.List, error)
 }
 
 type allAtOnceEvaluator struct {
 	treeNavigator DataTreeNavigator
-	treeCreator   PathTreeCreator
+	treeCreator   ExpressionParser
 }
 
 func NewAllAtOnceEvaluator() Evaluator {
-	return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewPathTreeCreator()}
+	return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewExpressionParser()}
+}
+
+func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error) {
+	inputCandidates := list.New()
+	for _, node := range nodes {
+		inputCandidates.PushBack(&CandidateNode{Node: node})
+	}
+	return e.EvaluateCandidateNodes(expression, inputCandidates)
+}
+
+func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCandidates *list.List) (*list.List, error) {
+	node, err := e.treeCreator.ParseExpression(expression)
+	if err != nil {
+		return nil, err
+	}
+	return e.treeNavigator.GetMatchingNodes(inputCandidates, node)
 }
 
 func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
 	fileIndex := 0
-	node, err := treeCreator.ParsePath(expression)
-	if err != nil {
-		return err
-	}
+
 	var allDocuments *list.List = list.New()
 	for _, filename := range filenames {
 		reader, err := readStream(filename)
@@ -37,7 +58,7 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 		allDocuments.PushBackList(fileDocuments)
 		fileIndex = fileIndex + 1
 	}
-	matches, err := treeNavigator.GetMatchingNodes(allDocuments, node)
+	matches, err := e.EvaluateCandidateNodes(expression, allDocuments)
 	if err != nil {
 		return err
 	}

pkg/yqlib/all_at_once_evaluator_test.go

@@ -0,0 +1,40 @@
+package yqlib
+
+import (
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+var evaluateNodesScenario = []expressionScenario{
+	{
+		document:   `a: hello`,
+		expression: `.a`,
+		expected: []string{
+			"D0, P[a], (!!str)::hello\n",
+		},
+	},
+	{
+		document:   `a: hello`,
+		expression: `.`,
+		expected: []string{
+			"D0, P[], (doc)::a: hello\n",
+		},
+	},
+	{
+		document:   `- a: "yes"`,
+		expression: `.[] | has("a")`,
+		expected: []string{
+			"D0, P[0], (!!bool)::true\n",
+		},
+	},
+}
+
+func TestAllAtOnceEvaluateNodes(t *testing.T) {
+	var evaluator = NewAllAtOnceEvaluator()
+	for _, tt := range evaluateNodesScenario {
+		node := test.ParseData(tt.document)
+		list, _ := evaluator.EvaluateNodes(tt.expression, &node)
+		test.AssertResultComplex(t, tt.expected, resultsToString(list))
+	}
+}

pkg/yqlib/candidate_node.go

@@ -21,7 +21,23 @@ func (n *CandidateNode) GetKey() string {
 	return fmt.Sprintf("%v - %v", n.Document, n.Path)
 }
 
-func (n *CandidateNode) CreateChildPath(path interface{}) []interface{} {
+func (n *CandidateNode) CreateChild(path interface{}, node *yaml.Node) *CandidateNode {
+	return &CandidateNode{
+		Node:      node,
+		Path:      n.createChildPath(path),
+		Document:  n.Document,
+		Filename:  n.Filename,
+		FileIndex: n.FileIndex,
+	}
+}
+
+func (n *CandidateNode) createChildPath(path interface{}) []interface{} {
+	if path == nil {
+		newPath := make([]interface{}, len(n.Path))
+		copy(newPath, n.Path)
+		return newPath
+	}
+
 	//don't use append as they may actually modify the path of the orignal node!
 	newPath := make([]interface{}, len(n.Path)+1)
 	copy(newPath, n.Path)
@@ -45,6 +61,7 @@ func (n *CandidateNode) UpdateFrom(other *CandidateNode) {
 	n.Node.Content = other.Node.Content
 	n.Node.Value = other.Node.Value
 	n.Node.Alias = other.Node.Alias
+	n.Node.Anchor = other.Node.Anchor
 }
 
 func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) {

pkg/yqlib/color_print.go

@@ -17,8 +17,8 @@ func format(attr color.Attribute) string {
 	return fmt.Sprintf("%s[%dm", escape, attr)
 }
 
-func ColorizeAndPrint(bytes []byte, writer io.Writer) error {
-	tokens := lexer.Tokenize(string(bytes))
+func colorizeAndPrint(yamlBytes []byte, writer io.Writer) error {
+	tokens := lexer.Tokenize(string(yamlBytes))
 	var p printer.Printer
 	p.Bool = func() *printer.Property {
 		return &printer.Property{

pkg/yqlib/data_tree_navigator.go

@@ -9,10 +9,10 @@ import (
 )
 
 type DataTreeNavigator interface {
-	// given a list of CandidateEntities and a pathNode,
-	// this will process the list against the given pathNode and return
+	// given a list of CandidateEntities and a expressionNode,
+	// this will process the list against the given expressionNode and return
 	// a new list of matching candidates
-	GetMatchingNodes(matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error)
+	GetMatchingNodes(matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error)
 }
 
 type dataTreeNavigator struct {
@@ -22,22 +22,22 @@ func NewDataTreeNavigator() DataTreeNavigator {
 	return &dataTreeNavigator{}
 }
 
-func (d *dataTreeNavigator) GetMatchingNodes(matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
-	if pathNode == nil {
+func (d *dataTreeNavigator) GetMatchingNodes(matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
+	if expressionNode == nil {
 		log.Debugf("getMatchingNodes - nothing to do")
 		return matchingNodes, nil
 	}
-	log.Debugf("Processing Op: %v", pathNode.Operation.toString())
+	log.Debugf("Processing Op: %v", expressionNode.Operation.toString())
 	if log.IsEnabledFor(logging.DEBUG) {
 		for el := matchingNodes.Front(); el != nil; el = el.Next() {
 			log.Debug(NodeToString(el.Value.(*CandidateNode)))
 		}
 	}
 	log.Debug(">>")
-	handler := pathNode.Operation.OperationType.Handler
+	handler := expressionNode.Operation.OperationType.Handler
 	if handler != nil {
-		return handler(d, matchingNodes, pathNode)
+		return handler(d, matchingNodes, expressionNode)
 	}
-	return nil, fmt.Errorf("Unknown operator %v", pathNode.Operation.OperationType)
+	return nil, fmt.Errorf("Unknown operator %v", expressionNode.Operation.OperationType)
 
 }

pkg/yqlib/doc/Add.md

@@ -1,9 +1,9 @@
 Add behaves differently according to the type of the LHS:
 - arrays: concatenate
-- number scalars: arithmetic addition (soon)
-- string scalars: concatenate (soon)
+- number scalars: arithmetic addition
+- string scalars: concatenate
 
-Use `+=` as append assign for things like increment. `.a += .x` is equivalent to running `.a |= . + .x`.
+Use `+=` as append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`.
 
 ## Concatenate and assign arrays
 Given a sample.yml file of:
@@ -67,23 +67,19 @@ will output
 - 2
 ```
 
-## Add object to array
+## Add new object to array
 Given a sample.yml file of:
 ```yaml
 a:
-  - 1
-  - 2
-c:
-  cat: meow
+  - dog: woof
 ```
 then
 ```bash
-yq eval '.a + .c' sample.yml
+yq eval '.a + {"cat": "meow"}' sample.yml
 ```
 will output
 ```yaml
-- 1
-- 2
+- dog: woof
 - cat: meow
 ```
 
@@ -131,3 +127,85 @@ b:
   - 4
 ```
 
+## String concatenation
+Given a sample.yml file of:
+```yaml
+a: cat
+b: meow
+```
+then
+```bash
+yq eval '.a = .a + .b' sample.yml
+```
+will output
+```yaml
+a: catmeow
+b: meow
+```
+
+## Relative string concatenation
+Given a sample.yml file of:
+```yaml
+a: cat
+b: meow
+```
+then
+```bash
+yq eval '.a += .b' sample.yml
+```
+will output
+```yaml
+a: catmeow
+b: meow
+```
+
+## Number addition - float
+If the lhs or rhs are floats then the expression will be calculated with floats.
+
+Given a sample.yml file of:
+```yaml
+a: 3
+b: 4.9
+```
+then
+```bash
+yq eval '.a = .a + .b' sample.yml
+```
+will output
+```yaml
+a: 7.9
+b: 4.9
+```
+
+## Number addition - int
+If both the lhs and rhs are ints then the expression will be calculated with ints.
+
+Given a sample.yml file of:
+```yaml
+a: 3
+b: 4
+```
+then
+```bash
+yq eval '.a = .a + .b' sample.yml
+```
+will output
+```yaml
+a: 7
+b: 4
+```
+
+## Increment number
+Given a sample.yml file of:
+```yaml
+a: 3
+```
+then
+```bash
+yq eval '.a += 1' sample.yml
+```
+will output
+```yaml
+a: 4
+```
+

pkg/yqlib/doc/Document Index.md

@@ -1,4 +1,4 @@
-Use the `documentIndex` operator to select nodes of a particular document.
+Use the `documentIndex` operator (or the `di` shorthand) to select nodes of a particular document.
 ## Retrieve a document index
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Env Variable Operators.md

@@ -1,3 +1,5 @@
+This operator is used to handle environment variables usage in path expressions. While environment variables can, of course, be passed in via your CLI with string interpolation, this often comes with complex quote escaping and can be tricky to write and read. Note that there are two forms, `env` which will parse the environment variable as a yaml (be it a map, array, string, number of boolean) and `strenv` which will always parse the argument as a string.
+
 
 ## Read string environment variable
 Running

pkg/yqlib/doc/Multiply.md

@@ -180,7 +180,7 @@ g: thongs
 f: *cat
 ```
 
-## Merge does not copy anchor names
+## Merge copies anchor names
 Given a sample.yml file of:
 ```yaml
 a:
@@ -197,7 +197,7 @@ yq eval '.c * .a' sample.yml
 will output
 ```yaml
 g: thongs
-c: frog
+c: &cat frog
 ```
 
 ## Merge with merge anchors

pkg/yqlib/doc/Recursive Descent (Glob).md

@@ -33,6 +33,8 @@ frog
 ```
 
 ## Recursively find nodes with keys
+Note that this example has wrapped the expression in `[]` to show that there are two matches returned. You do not have to wrap in `[]` in your path expression.
+
 Given a sample.yml file of:
 ```yaml
 a:
@@ -43,16 +45,16 @@ a:
 ```
 then
 ```bash
-yq eval '.. | select(has("name"))' sample.yml
+yq eval '[.. | select(has("name"))]' sample.yml
 ```
 will output
 ```yaml
-name: frog
-b:
-  name: blog
+- name: frog
+  b:
+    name: blog
+    age: 12
+- name: blog
   age: 12
-name: blog
-age: 12
 ```
 
 ## Recursively find nodes with values

pkg/yqlib/doc/headers/Add.md

@@ -1,6 +1,6 @@
 Add behaves differently according to the type of the LHS:
 - arrays: concatenate
-- number scalars: arithmetic addition (soon)
-- string scalars: concatenate (soon)
+- number scalars: arithmetic addition
+- string scalars: concatenate
 
-Use `+=` as append assign for things like increment. `.a += .x` is equivalent to running `.a |= . + .x`.
+Use `+=` as append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`.

pkg/yqlib/doc/headers/Document Index.md

@@ -1 +1 @@
-Use the `documentIndex` operator to select nodes of a particular document.
+Use the `documentIndex` operator (or the `di` shorthand) to select nodes of a particular document.

pkg/yqlib/doc/headers/Env Variable Operators.md

@@ -0,0 +1,2 @@
+This operator is used to handle environment variables usage in path expressions. While environment variables can, of course, be passed in via your CLI with string interpolation, this often comes with complex quote escaping and can be tricky to write and read. Note that there are two forms, `env` which will parse the environment variable as a yaml (be it a map, array, string, number of boolean) and `strenv` which will always parse the argument as a string.
+

pkg/yqlib/encoder.go

@@ -50,7 +50,7 @@ func (ye *yamlEncoder) Encode(node *yaml.Node) error {
 	}
 
 	if ye.colorise {
-		return ColorizeAndPrint(tempBuffer.Bytes(), ye.destination)
+		return colorizeAndPrint(tempBuffer.Bytes(), ye.destination)
 	}
 	return nil
 }

pkg/yqlib/expression_parser.go

@@ -5,29 +5,28 @@ import (
 	"strings"
 )
 
-var myPathTokeniser = NewPathTokeniser()
-var myPathPostfixer = NewPathPostFixer()
+var myPathTokeniser = newExpressionTokeniser()
+var myPathPostfixer = newExpressionPostFixer()
 
-type PathTreeNode struct {
+type ExpressionNode struct {
 	Operation *Operation
-	Lhs       *PathTreeNode
-	Rhs       *PathTreeNode
+	Lhs       *ExpressionNode
+	Rhs       *ExpressionNode
 }
 
-type PathTreeCreator interface {
-	ParsePath(path string) (*PathTreeNode, error)
-	CreatePathTree(postFixPath []*Operation) (*PathTreeNode, error)
+type ExpressionParser interface {
+	ParseExpression(expression string) (*ExpressionNode, error)
 }
 
-type pathTreeCreator struct {
+type expressionParserImpl struct {
 }
 
-func NewPathTreeCreator() PathTreeCreator {
-	return &pathTreeCreator{}
+func NewExpressionParser() ExpressionParser {
+	return &expressionParserImpl{}
 }
 
-func (p *pathTreeCreator) ParsePath(path string) (*PathTreeNode, error) {
-	tokens, err := myPathTokeniser.Tokenise(path)
+func (p *expressionParserImpl) ParseExpression(expression string) (*ExpressionNode, error) {
+	tokens, err := myPathTokeniser.Tokenise(expression)
 	if err != nil {
 		return nil, err
 	}
@@ -36,18 +35,18 @@ func (p *pathTreeCreator) ParsePath(path string) (*PathTreeNode, error) {
 	if err != nil {
 		return nil, err
 	}
-	return p.CreatePathTree(Operations)
+	return p.createExpressionTree(Operations)
 }
 
-func (p *pathTreeCreator) CreatePathTree(postFixPath []*Operation) (*PathTreeNode, error) {
-	var stack = make([]*PathTreeNode, 0)
+func (p *expressionParserImpl) createExpressionTree(postFixPath []*Operation) (*ExpressionNode, error) {
+	var stack = make([]*ExpressionNode, 0)
 
 	if len(postFixPath) == 0 {
 		return nil, nil
 	}
 
 	for _, Operation := range postFixPath {
-		var newNode = PathTreeNode{Operation: Operation}
+		var newNode = ExpressionNode{Operation: Operation}
 		log.Debugf("pathTree %v ", Operation.toString())
 		if Operation.OperationType.NumArgs > 0 {
 			numArgs := Operation.OperationType.NumArgs

pkg/yqlib/expression_parser_test.go

@@ -7,36 +7,36 @@ import (
 )
 
 func TestPathTreeNoArgsForTwoArgOp(t *testing.T) {
-	_, err := treeCreator.ParsePath("=")
+	_, err := NewExpressionParser().ParseExpression("=")
 	test.AssertResultComplex(t, "'=' expects 2 args but there is 0", err.Error())
 }
 
 func TestPathTreeOneLhsArgsForTwoArgOp(t *testing.T) {
-	_, err := treeCreator.ParsePath(".a =")
+	_, err := NewExpressionParser().ParseExpression(".a =")
 	test.AssertResultComplex(t, "'=' expects 2 args but there is 1", err.Error())
 }
 
 func TestPathTreeOneRhsArgsForTwoArgOp(t *testing.T) {
-	_, err := treeCreator.ParsePath("= .a")
+	_, err := NewExpressionParser().ParseExpression("= .a")
 	test.AssertResultComplex(t, "'=' expects 2 args but there is 1", err.Error())
 }
 
 func TestPathTreeTwoArgsForTwoArgOp(t *testing.T) {
-	_, err := treeCreator.ParsePath(".a = .b")
+	_, err := NewExpressionParser().ParseExpression(".a = .b")
 	test.AssertResultComplex(t, nil, err)
 }
 
 func TestPathTreeNoArgsForOneArgOp(t *testing.T) {
-	_, err := treeCreator.ParsePath("explode")
+	_, err := NewExpressionParser().ParseExpression("explode")
 	test.AssertResultComplex(t, "'explode' expects 1 arg but received none", err.Error())
 }
 
 func TestPathTreeOneArgForOneArgOp(t *testing.T) {
-	_, err := treeCreator.ParsePath("explode(.)")
+	_, err := NewExpressionParser().ParseExpression("explode(.)")
 	test.AssertResultComplex(t, nil, err)
 }
 
 func TestPathTreeExtraArgs(t *testing.T) {
-	_, err := treeCreator.ParsePath("sortKeys(.) explode(.)")
+	_, err := NewExpressionParser().ParseExpression("sortKeys(.) explode(.)")
 	test.AssertResultComplex(t, "expected end of expression but found 'explode', please check expression syntax", err.Error())
 }

pkg/yqlib/expression_postfix.go

@@ -6,40 +6,40 @@ import (
 	logging "gopkg.in/op/go-logging.v1"
 )
 
-type PathPostFixer interface {
-	ConvertToPostfix([]*Token) ([]*Operation, error)
+type expressionPostFixer interface {
+	ConvertToPostfix([]*token) ([]*Operation, error)
 }
 
-type pathPostFixer struct {
+type expressionPostFixerImpl struct {
 }
 
-func NewPathPostFixer() PathPostFixer {
-	return &pathPostFixer{}
+func newExpressionPostFixer() expressionPostFixer {
+	return &expressionPostFixerImpl{}
 }
 
-func popOpToResult(opStack []*Token, result []*Operation) ([]*Token, []*Operation) {
-	var newOp *Token
+func popOpToResult(opStack []*token, result []*Operation) ([]*token, []*Operation) {
+	var newOp *token
 	opStack, newOp = opStack[0:len(opStack)-1], opStack[len(opStack)-1]
 	return opStack, append(result, newOp.Operation)
 }
 
-func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, error) {
+func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Operation, error) {
 	var result []*Operation
 	// surround the whole thing with quotes
-	var opStack = []*Token{&Token{TokenType: OpenBracket}}
-	var tokens = append(infixTokens, &Token{TokenType: CloseBracket})
+	var opStack = []*token{&token{TokenType: openBracket}}
+	var tokens = append(infixTokens, &token{TokenType: closeBracket})
 
-	for _, token := range tokens {
-		log.Debugf("postfix processing token %v, %v", token.toString(), token.Operation)
-		switch token.TokenType {
-		case OpenBracket, OpenCollect, OpenCollectObject:
-			opStack = append(opStack, token)
-		case CloseCollect, CloseCollectObject:
-			var opener TokenType = OpenCollect
-			var collectOperator *OperationType = Collect
-			if token.TokenType == CloseCollectObject {
-				opener = OpenCollectObject
-				collectOperator = CollectObject
+	for _, currentToken := range tokens {
+		log.Debugf("postfix processing currentToken %v, %v", currentToken.toString(), currentToken.Operation)
+		switch currentToken.TokenType {
+		case openBracket, openCollect, openCollectObject:
+			opStack = append(opStack, currentToken)
+		case closeCollect, closeCollectObject:
+			var opener tokenType = openCollect
+			var collectOperator *operationType = collectOpType
+			if currentToken.TokenType == closeCollectObject {
+				opener = openCollectObject
+				collectOperator = collectObjectOpType
 			}
 			itemsInMiddle := false
 			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != opener {
@@ -48,7 +48,7 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, er
 			}
 			if !itemsInMiddle {
 				// must be an empty collection, add the empty object as a LHS parameter
-				result = append(result, &Operation{OperationType: Empty})
+				result = append(result, &Operation{OperationType: emptyOpType})
 			}
 			if len(opStack) == 0 {
 				return nil, errors.New("Bad path expression, got close collect brackets without matching opening bracket")
@@ -56,10 +56,10 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, er
 			// now we should have [] as the last element on the opStack, get rid of it
 			opStack = opStack[0 : len(opStack)-1]
 			//and append a collect to the opStack
-			opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: ShortPipe}})
-			opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: collectOperator}})
-		case CloseBracket:
-			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != OpenBracket {
+			opStack = append(opStack, &token{TokenType: operationToken, Operation: &Operation{OperationType: shortPipeOpType}})
+			opStack = append(opStack, &token{TokenType: operationToken, Operation: &Operation{OperationType: collectOperator}})
+		case closeBracket:
+			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != openBracket {
 				opStack, result = popOpToResult(opStack, result)
 			}
 			if len(opStack) == 0 {
@@ -69,22 +69,22 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, er
 			opStack = opStack[0 : len(opStack)-1]
 
 		default:
-			var currentPrecedence = token.Operation.OperationType.Precedence
+			var currentPrecedence = currentToken.Operation.OperationType.Precedence
 			// pop off higher precedent operators onto the result
 			for len(opStack) > 0 &&
-				opStack[len(opStack)-1].TokenType == OperationToken &&
+				opStack[len(opStack)-1].TokenType == operationToken &&
 				opStack[len(opStack)-1].Operation.OperationType.Precedence >= currentPrecedence {
 				opStack, result = popOpToResult(opStack, result)
 			}
 			// add this operator to the opStack
-			opStack = append(opStack, token)
+			opStack = append(opStack, currentToken)
 		}
 	}
 
 	if log.IsEnabledFor(logging.DEBUG) {
 		log.Debugf("PostFix Result:")
-		for _, token := range result {
-			log.Debugf("> %v", token.toString())
+		for _, currentToken := range result {
+			log.Debugf("> %v", currentToken.toString())
 		}
 	}
 

pkg/yqlib/expression_processing_test.go

@@ -164,8 +164,8 @@ var pathTests = []struct {
 	},
 }
 
-var tokeniser = NewPathTokeniser()
-var postFixer = NewPathPostFixer()
+var tokeniser = newExpressionTokeniser()
+var postFixer = newExpressionPostFixer()
 
 func TestPathParsing(t *testing.T) {
 	for _, tt := range pathTests {

pkg/yqlib/expression_tokeniser.go

@@ -0,0 +1,407 @@
+package yqlib
+
+import (
+	"fmt"
+	"strconv"
+
+	lex "github.com/timtadh/lexmachine"
+	"github.com/timtadh/lexmachine/machines"
+)
+
+func skip(*lex.Scanner, *machines.Match) (interface{}, error) {
+	return nil, nil
+}
+
+type tokenType uint32
+
+const (
+	operationToken = 1 << iota
+	openBracket
+	closeBracket
+	openCollect
+	closeCollect
+	openCollectObject
+	closeCollectObject
+	traverseArrayCollect
+)
+
+type token struct {
+	TokenType            tokenType
+	Operation            *Operation
+	AssignOperation      *Operation // e.g. tag (GetTag) op becomes AssignTag if '=' follows it
+	CheckForPostTraverse bool       // e.g. [1]cat should really be [1].cat
+
+}
+
+func (t *token) toString() string {
+	if t.TokenType == operationToken {
+		log.Debug("toString, its an op")
+		return t.Operation.toString()
+	} else if t.TokenType == openBracket {
+		return "("
+	} else if t.TokenType == closeBracket {
+		return ")"
+	} else if t.TokenType == openCollect {
+		return "["
+	} else if t.TokenType == closeCollect {
+		return "]"
+	} else if t.TokenType == openCollectObject {
+		return "{"
+	} else if t.TokenType == closeCollectObject {
+		return "}"
+	} else if t.TokenType == traverseArrayCollect {
+		return ".["
+
+	} else {
+		return "NFI"
+	}
+}
+
+func pathToken(wrapped bool) lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		value := string(m.Bytes)
+		value = value[1:]
+		if wrapped {
+			value = unwrap(value)
+		}
+		log.Debug("PathToken %v", value)
+		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value}
+		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
+	}
+}
+
+func documentToken() lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		var numberString = string(m.Bytes)
+		numberString = numberString[1:]
+		var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) // nolint
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+		log.Debug("documentToken %v", string(m.Bytes))
+		op := &Operation{OperationType: documentFilterOpType, Value: number, StringValue: numberString}
+		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
+	}
+}
+
+func opToken(op *operationType) lex.Action {
+	return opTokenWithPrefs(op, nil, nil)
+}
+
+func opAssignableToken(opType *operationType, assignOpType *operationType) lex.Action {
+	return opTokenWithPrefs(opType, assignOpType, nil)
+}
+
+func assignOpToken(updateAssign bool) lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		log.Debug("assignOpToken %v", string(m.Bytes))
+		value := string(m.Bytes)
+		op := &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, UpdateAssign: updateAssign}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func opTokenWithPrefs(op *operationType, assignOpType *operationType, preferences interface{}) lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		log.Debug("opTokenWithPrefs %v", string(m.Bytes))
+		value := string(m.Bytes)
+		op := &Operation{OperationType: op, Value: op.Type, StringValue: value, Preferences: preferences}
+		var assign *Operation
+		if assignOpType != nil {
+			assign = &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, Preferences: preferences}
+		}
+		return &token{TokenType: operationToken, Operation: op, AssignOperation: assign}, nil
+	}
+}
+
+func assignAllCommentsOp(updateAssign bool) lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		log.Debug("assignAllCommentsOp %v", string(m.Bytes))
+		value := string(m.Bytes)
+		op := &Operation{
+			OperationType: assignCommentOpType,
+			Value:         assignCommentOpType.Type,
+			StringValue:   value,
+			UpdateAssign:  updateAssign,
+			Preferences:   &commentOpPreferences{LineComment: true, HeadComment: true, FootComment: true},
+		}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func literalToken(pType tokenType, checkForPost bool) lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		return &token{TokenType: pType, CheckForPostTraverse: checkForPost}, nil
+	}
+}
+
+func unwrap(value string) string {
+	return value[1 : len(value)-1]
+}
+
+func numberValue() lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		var numberString = string(m.Bytes)
+		var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) // nolint
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		return &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil
+	}
+}
+
+func floatValue() lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		var numberString = string(m.Bytes)
+		var number, errParsingInt = strconv.ParseFloat(numberString, 64) // nolint
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+		return &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil
+	}
+}
+
+func booleanValue(val bool) lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		return &token{TokenType: operationToken, Operation: createValueOperation(val, string(m.Bytes))}, nil
+	}
+}
+
+func stringValue(wrapped bool) lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		value := string(m.Bytes)
+		if wrapped {
+			value = unwrap(value)
+		}
+		return &token{TokenType: operationToken, Operation: createValueOperation(value, value)}, nil
+	}
+}
+
+func envOp(strenv bool) lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		value := string(m.Bytes)
+		preferences := &envOpPreferences{}
+
+		if strenv {
+			// strenv( )
+			value = value[7 : len(value)-1]
+			preferences.StringValue = true
+		} else {
+			//env( )
+			value = value[4 : len(value)-1]
+		}
+
+		envOperation := createValueOperation(value, value)
+		envOperation.OperationType = envOpType
+		envOperation.Preferences = preferences
+
+		return &token{TokenType: operationToken, Operation: envOperation}, nil
+	}
+}
+
+func nullValue() lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		return &token{TokenType: operationToken, Operation: createValueOperation(nil, string(m.Bytes))}, nil
+	}
+}
+
+func selfToken() lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		op := &Operation{OperationType: selfReferenceOpType}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func initLexer() (*lex.Lexer, error) {
+	lexer := lex.NewLexer()
+	lexer.Add([]byte(`\(`), literalToken(openBracket, false))
+	lexer.Add([]byte(`\)`), literalToken(closeBracket, true))
+
+	lexer.Add([]byte(`\.\[`), literalToken(traverseArrayCollect, false))
+	lexer.Add([]byte(`\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, &recursiveDescentPreferences{RecurseArray: true,
+		TraversePreferences: &traversePreferences{FollowAlias: false, IncludeMapKeys: false}}))
+
+	lexer.Add([]byte(`\.\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, &recursiveDescentPreferences{RecurseArray: true,
+		TraversePreferences: &traversePreferences{FollowAlias: false, IncludeMapKeys: true}}))
+
+	lexer.Add([]byte(`,`), opToken(unionOpType))
+	lexer.Add([]byte(`:\s*`), opToken(createMapOpType))
+	lexer.Add([]byte(`length`), opToken(lengthOpType))
+	lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType))
+	lexer.Add([]byte(`select`), opToken(selectOpType))
+	lexer.Add([]byte(`has`), opToken(hasOpType))
+	lexer.Add([]byte(`explode`), opToken(explodeOpType))
+	lexer.Add([]byte(`or`), opToken(orOpType))
+	lexer.Add([]byte(`and`), opToken(andOpType))
+	lexer.Add([]byte(`not`), opToken(notOpType))
+	lexer.Add([]byte(`\/\/`), opToken(alternativeOpType))
+
+	lexer.Add([]byte(`documentIndex`), opToken(getDocumentIndexOpType))
+	lexer.Add([]byte(`di`), opToken(getDocumentIndexOpType))
+
+	lexer.Add([]byte(`style`), opAssignableToken(getStyleOpType, assignStyleOpType))
+
+	lexer.Add([]byte(`tag`), opAssignableToken(getTagOpType, assignTagOpType))
+	lexer.Add([]byte(`anchor`), opAssignableToken(getAnchorOpType, assignAnchorOpType))
+	lexer.Add([]byte(`alias`), opAssignableToken(getAliasOptype, assignAliasOpType))
+	lexer.Add([]byte(`filename`), opToken(getFilenameOpType))
+	lexer.Add([]byte(`fileIndex`), opToken(getFileIndexOpType))
+	lexer.Add([]byte(`fi`), opToken(getFileIndexOpType))
+	lexer.Add([]byte(`path`), opToken(getPathOpType))
+
+	lexer.Add([]byte(`lineComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, &commentOpPreferences{LineComment: true}))
+
+	lexer.Add([]byte(`headComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, &commentOpPreferences{HeadComment: true}))
+
+	lexer.Add([]byte(`footComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, &commentOpPreferences{FootComment: true}))
+
+	lexer.Add([]byte(`comments\s*=`), assignAllCommentsOp(false))
+	lexer.Add([]byte(`comments\s*\|=`), assignAllCommentsOp(true))
+
+	lexer.Add([]byte(`collect`), opToken(collectOpType))
+
+	lexer.Add([]byte(`\s*==\s*`), opToken(equalsOpType))
+	lexer.Add([]byte(`\s*=\s*`), assignOpToken(false))
+
+	lexer.Add([]byte(`del`), opToken(deleteChildOpType))
+
+	lexer.Add([]byte(`\s*\|=\s*`), assignOpToken(true))
+
+	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
+
+	lexer.Add([]byte(`d[0-9]+`), documentToken())
+	lexer.Add([]byte(`\."[^ "]+"`), pathToken(true))
+	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+`), pathToken(false))
+	lexer.Add([]byte(`\.`), selfToken())
+
+	lexer.Add([]byte(`\|`), opToken(pipeOpType))
+
+	lexer.Add([]byte(`-?\d+(\.\d+)`), floatValue())
+	lexer.Add([]byte(`-?[1-9](\.\d+)?[Ee][-+]?\d+`), floatValue())
+	lexer.Add([]byte(`-?\d+`), numberValue())
+
+	lexer.Add([]byte(`[Tt][Rr][Uu][Ee]`), booleanValue(true))
+	lexer.Add([]byte(`[Ff][Aa][Ll][Ss][Ee]`), booleanValue(false))
+
+	lexer.Add([]byte(`[Nn][Uu][Ll][Ll]`), nullValue())
+	lexer.Add([]byte(`~`), nullValue())
+
+	lexer.Add([]byte(`"[^"]*"`), stringValue(true))
+	lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true))
+	lexer.Add([]byte(`env\([^\)]+\)`), envOp(false))
+
+	lexer.Add([]byte(`\[`), literalToken(openCollect, false))
+	lexer.Add([]byte(`\]`), literalToken(closeCollect, true))
+	lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
+	lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
+	lexer.Add([]byte(`\*`), opTokenWithPrefs(multiplyOpType, nil, &multiplyPreferences{AppendArrays: false}))
+	lexer.Add([]byte(`\*\+`), opTokenWithPrefs(multiplyOpType, nil, &multiplyPreferences{AppendArrays: true}))
+	lexer.Add([]byte(`\+`), opToken(addOpType))
+	lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
+
+	err := lexer.Compile()
+	if err != nil {
+		return nil, err
+	}
+	return lexer, nil
+}
+
+type expressionTokeniser interface {
+	Tokenise(expression string) ([]*token, error)
+}
+
+type expressionTokeniserImpl struct {
+	lexer *lex.Lexer
+}
+
+func newExpressionTokeniser() expressionTokeniser {
+	var lexer, err = initLexer()
+	if err != nil {
+		panic(err)
+	}
+	return &expressionTokeniserImpl{lexer}
+}
+
+func (p *expressionTokeniserImpl) Tokenise(expression string) ([]*token, error) {
+	scanner, err := p.lexer.Scanner([]byte(expression))
+
+	if err != nil {
+		return nil, fmt.Errorf("Parsing expression: %v", err)
+	}
+	var tokens []*token
+	for tok, err, eof := scanner.Next(); !eof; tok, err, eof = scanner.Next() {
+
+		if tok != nil {
+			currentToken := tok.(*token)
+			log.Debugf("Tokenising %v", currentToken.toString())
+			tokens = append(tokens, currentToken)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("Parsing expression: %v", err)
+		}
+	}
+	var postProcessedTokens = make([]*token, 0)
+
+	skipNextToken := false
+
+	for index := range tokens {
+		if skipNextToken {
+			skipNextToken = false
+		} else {
+			postProcessedTokens, skipNextToken = p.handleToken(tokens, index, postProcessedTokens)
+		}
+	}
+
+	return postProcessedTokens, nil
+}
+
+func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postProcessedTokens []*token) (tokensAccum []*token, skipNextToken bool) {
+	skipNextToken = false
+	currentToken := tokens[index]
+
+	if currentToken.TokenType == traverseArrayCollect {
+		//need to put a traverse array then a collect currentToken
+		// do this by adding traverse then converting currentToken to collect
+
+		op := &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+
+		currentToken = &token{TokenType: openCollect}
+
+	}
+
+	if index != len(tokens)-1 && currentToken.AssignOperation != nil &&
+		tokens[index+1].TokenType == operationToken &&
+		tokens[index+1].Operation.OperationType == assignOpType {
+		currentToken.Operation = currentToken.AssignOperation
+		currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign
+		skipNextToken = true
+	}
+
+	postProcessedTokens = append(postProcessedTokens, currentToken)
+
+	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
+		tokens[index+1].TokenType == operationToken &&
+		tokens[index+1].Operation.OperationType == traversePathOpType {
+		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+	}
+	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
+		tokens[index+1].TokenType == openCollect {
+
+		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+
+		op = &Operation{OperationType: traverseArrayOpType}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+	}
+	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
+		tokens[index+1].TokenType == traverseArrayCollect {
+
+		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+
+	}
+	return postProcessedTokens, skipNextToken
+}

pkg/yqlib/lib.go

@@ -1,3 +1,5 @@
+// Use the top level Evaluator or StreamEvaluator to evaluate expressions and return matches.
+//
 package yqlib
 
 import (
@@ -11,86 +13,85 @@ import (
 
 var log = logging.MustGetLogger("yq-lib")
 
-type OperationType struct {
+type operationType struct {
 	Type       string
 	NumArgs    uint // number of arguments to the op
 	Precedence uint
-	Handler    OperatorHandler
+	Handler    operatorHandler
 }
 
 // operators TODO:
-// - keys operator for controlling key metadata (particularly anchors/aliases)
 // - mergeEmpty (sets only if the document is empty, do I do that now?)
 
-var Or = &OperationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: OrOperator}
-var And = &OperationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: AndOperator}
+var orOpType = &operationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: orOperator}
+var andOpType = &operationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: andOperator}
 
-var Union = &OperationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: UnionOperator}
+var unionOpType = &operationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: unionOperator}
 
-var Pipe = &OperationType{Type: "PIPE", NumArgs: 2, Precedence: 30, Handler: PipeOperator}
+var pipeOpType = &operationType{Type: "PIPE", NumArgs: 2, Precedence: 30, Handler: pipeOperator}
 
-var Assign = &OperationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: AssignUpdateOperator}
-var AddAssign = &OperationType{Type: "ADD_ASSIGN", NumArgs: 2, Precedence: 40, Handler: AddAssignOperator}
+var assignOpType = &operationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: assignUpdateOperator}
+var addAssignOpType = &operationType{Type: "ADD_ASSIGN", NumArgs: 2, Precedence: 40, Handler: addAssignOperator}
 
-var AssignAttributes = &OperationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: AssignAttributesOperator}
-var AssignStyle = &OperationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: AssignStyleOperator}
-var AssignTag = &OperationType{Type: "ASSIGN_TAG", NumArgs: 2, Precedence: 40, Handler: AssignTagOperator}
-var AssignComment = &OperationType{Type: "ASSIGN_COMMENT", NumArgs: 2, Precedence: 40, Handler: AssignCommentsOperator}
-var AssignAnchor = &OperationType{Type: "ASSIGN_ANCHOR", NumArgs: 2, Precedence: 40, Handler: AssignAnchorOperator}
-var AssignAlias = &OperationType{Type: "ASSIGN_ALIAS", NumArgs: 2, Precedence: 40, Handler: AssignAliasOperator}
+var assignAttributesOpType = &operationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: assignAttributesOperator}
+var assignStyleOpType = &operationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: assignStyleOperator}
+var assignTagOpType = &operationType{Type: "ASSIGN_TAG", NumArgs: 2, Precedence: 40, Handler: assignTagOperator}
+var assignCommentOpType = &operationType{Type: "ASSIGN_COMMENT", NumArgs: 2, Precedence: 40, Handler: assignCommentsOperator}
+var assignAnchorOpType = &operationType{Type: "ASSIGN_ANCHOR", NumArgs: 2, Precedence: 40, Handler: assignAnchorOperator}
+var assignAliasOpType = &operationType{Type: "ASSIGN_ALIAS", NumArgs: 2, Precedence: 40, Handler: assignAliasOperator}
 
-var Multiply = &OperationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 45, Handler: MultiplyOperator}
-var Add = &OperationType{Type: "ADD", NumArgs: 2, Precedence: 45, Handler: AddOperator}
-var Alternative = &OperationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 45, Handler: AlternativeOperator}
+var multiplyOpType = &operationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 45, Handler: multiplyOperator}
+var addOpType = &operationType{Type: "ADD", NumArgs: 2, Precedence: 45, Handler: addOperator}
+var alternativeOpType = &operationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 45, Handler: alternativeOperator}
 
-var Equals = &OperationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: EqualsOperator}
-var CreateMap = &OperationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 40, Handler: CreateMapOperator}
+var equalsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: equalsOperator}
+var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 40, Handler: createMapOperator}
 
-var ShortPipe = &OperationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: 45, Handler: PipeOperator}
+var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: 45, Handler: pipeOperator}
 
-var Length = &OperationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: LengthOperator}
-var Collect = &OperationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: CollectOperator}
-var GetStyle = &OperationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: GetStyleOperator}
-var GetTag = &OperationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: GetTagOperator}
-var GetComment = &OperationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: GetCommentsOperator}
-var GetAnchor = &OperationType{Type: "GET_ANCHOR", NumArgs: 0, Precedence: 50, Handler: GetAnchorOperator}
-var GetAlias = &OperationType{Type: "GET_ALIAS", NumArgs: 0, Precedence: 50, Handler: GetAliasOperator}
-var GetDocumentIndex = &OperationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: GetDocumentIndexOperator}
-var GetFilename = &OperationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: GetFilenameOperator}
-var GetFileIndex = &OperationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: GetFileIndexOperator}
-var GetPath = &OperationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50, Handler: GetPathOperator}
+var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
+var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: collectOperator}
+var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}
+var getTagOpType = &operationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: getTagOperator}
+var getCommentOpType = &operationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: getCommentsOperator}
+var getAnchorOpType = &operationType{Type: "GET_ANCHOR", NumArgs: 0, Precedence: 50, Handler: getAnchorOperator}
+var getAliasOptype = &operationType{Type: "GET_ALIAS", NumArgs: 0, Precedence: 50, Handler: getAliasOperator}
+var getDocumentIndexOpType = &operationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: getDocumentIndexOperator}
+var getFilenameOpType = &operationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: getFilenameOperator}
+var getFileIndexOpType = &operationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: getFileIndexOperator}
+var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50, Handler: getPathOperator}
 
-var Explode = &OperationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: ExplodeOperator}
-var SortKeys = &OperationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: SortKeysOperator}
+var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator}
+var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
 
-var CollectObject = &OperationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: CollectObjectOperator}
-var TraversePath = &OperationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
-var TraverseArray = &OperationType{Type: "TRAVERSE_ARRAY", NumArgs: 1, Precedence: 50, Handler: TraverseArrayOperator}
+var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}
+var traversePathOpType = &operationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: traversePathOperator}
+var traverseArrayOpType = &operationType{Type: "TRAVERSE_ARRAY", NumArgs: 1, Precedence: 50, Handler: traverseArrayOperator}
 
-var DocumentFilter = &OperationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
-var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: SelfOperator}
-var ValueOp = &OperationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: ValueOperator}
-var EnvOp = &OperationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: EnvOperator}
-var Not = &OperationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: NotOperator}
-var Empty = &OperationType{Type: "EMPTY", NumArgs: 50, Handler: EmptyOperator}
+var documentFilterOpType = &operationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: traversePathOperator}
+var selfReferenceOpType = &operationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: selfOperator}
+var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: valueOperator}
+var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator}
+var notOpType = &operationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: notOperator}
+var emptyOpType = &operationType{Type: "EMPTY", NumArgs: 50, Handler: emptyOperator}
 
-var RecursiveDescent = &OperationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: RecursiveDescentOperator}
+var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: recursiveDescentOperator}
 
-var Select = &OperationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: SelectOperator}
-var Has = &OperationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: HasOperator}
-var DeleteChild = &OperationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: DeleteChildOperator}
-var DeleteImmediateChild = &OperationType{Type: "DELETE_IMMEDIATE_CHILD", NumArgs: 1, Precedence: 40, Handler: DeleteImmediateChildOperator}
+var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator}
+var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator}
+var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
+var deleteImmediateChildOpType = &operationType{Type: "DELETE_IMMEDIATE_CHILD", NumArgs: 1, Precedence: 40, Handler: deleteImmediateChildOperator}
 
 type Operation struct {
-	OperationType *OperationType
+	OperationType *operationType
 	Value         interface{}
 	StringValue   string
 	CandidateNode *CandidateNode // used for Value Path elements
 	Preferences   interface{}
-	UpdateAssign  bool // used for assign ops, when true it means we evaluate the rhs given the lhs (instead of matching nodes)
+	UpdateAssign  bool // used for assign ops, when true it means we evaluate the rhs given the lhs
 }
 
-func CreateValueOperation(value interface{}, stringValue string) *Operation {
+func createValueOperation(value interface{}, stringValue string) *Operation {
 	var node yaml.Node = yaml.Node{Kind: yaml.ScalarNode}
 	node.Value = stringValue
 
@@ -108,7 +109,7 @@ func CreateValueOperation(value interface{}, stringValue string) *Operation {
 	}
 
 	return &Operation{
-		OperationType: ValueOp,
+		OperationType: valueOpType,
 		Value:         value,
 		StringValue:   stringValue,
 		CandidateNode: &CandidateNode{Node: &node},
@@ -117,13 +118,13 @@ func CreateValueOperation(value interface{}, stringValue string) *Operation {
 
 // debugging purposes only
 func (p *Operation) toString() string {
-	if p.OperationType == TraversePath {
+	if p.OperationType == traversePathOpType {
 		return fmt.Sprintf("%v", p.Value)
-	} else if p.OperationType == DocumentFilter {
+	} else if p.OperationType == documentFilterOpType {
 		return fmt.Sprintf("d%v", p.Value)
-	} else if p.OperationType == SelfReference {
+	} else if p.OperationType == selfReferenceOpType {
 		return "SELF"
-	} else if p.OperationType == ValueOp {
+	} else if p.OperationType == valueOpType {
 		return fmt.Sprintf("%v (%T)", p.Value, p.Value)
 	} else {
 		return fmt.Sprintf("%v", p.OperationType.Type)

pkg/yqlib/matchKeyString.go

@@ -1,6 +1,6 @@
 package yqlib
 
-func Match(name string, pattern string) (matched bool) {
+func matchKey(name string, pattern string) (matched bool) {
 	if pattern == "" {
 		return name == pattern
 	}

pkg/yqlib/operator_add.go

@@ -4,21 +4,22 @@ import (
 	"fmt"
 
 	"container/list"
+	"strconv"
 
 	yaml "gopkg.in/yaml.v3"
 )
 
-func createSelfAddOp(rhs *PathTreeNode) *PathTreeNode {
-	return &PathTreeNode{Operation: &Operation{OperationType: Add},
-		Lhs: &PathTreeNode{Operation: &Operation{OperationType: SelfReference}},
+func createAddOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {
+	return &ExpressionNode{Operation: &Operation{OperationType: addOpType},
+		Lhs: lhs,
 		Rhs: rhs}
 }
 
-func AddAssignOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
-	assignmentOp := &Operation{OperationType: Assign}
-	assignmentOp.UpdateAssign = true
+func addAssignOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
+	assignmentOp := &Operation{OperationType: assignOpType}
+	assignmentOp.UpdateAssign = false
 
-	assignmentOpNode := &PathTreeNode{Operation: assignmentOp, Lhs: pathNode.Lhs, Rhs: createSelfAddOp(pathNode.Rhs)}
+	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createAddOp(expressionNode.Lhs, expressionNode.Rhs)}
 	return d.GetMatchingNodes(matchingNodes, assignmentOpNode)
 }
 
@@ -36,22 +37,17 @@ func toNodes(candidate *CandidateNode) []*yaml.Node {
 
 }
 
-func AddOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func addOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("Add operator")
 
-	return crossFunction(d, matchingNodes, pathNode, add)
+	return crossFunction(d, matchingNodes, expressionNode, add)
 }
 
 func add(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-	lhs.Node = UnwrapDoc(lhs.Node)
-	rhs.Node = UnwrapDoc(rhs.Node)
+	lhs.Node = unwrapDoc(lhs.Node)
+	rhs.Node = unwrapDoc(rhs.Node)
 
-	target := &CandidateNode{
-		Path:     lhs.Path,
-		Document: lhs.Document,
-		Filename: lhs.Filename,
-		Node:     &yaml.Node{},
-	}
+	target := lhs.CreateChild(nil, &yaml.Node{})
 	lhsNode := lhs.Node
 
 	switch lhsNode.Kind {
@@ -63,7 +59,48 @@ func add(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*Candida
 		target.Node.Tag = "!!seq"
 		target.Node.Content = append(lhsNode.Content, toNodes(rhs)...)
 	case yaml.ScalarNode:
-		return nil, fmt.Errorf("Scalars not yet supported for addition")
+		if rhs.Node.Kind != yaml.ScalarNode {
+			return nil, fmt.Errorf("%v (%v) cannot be added to a %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag)
+		}
+		target.Node.Kind = yaml.ScalarNode
+		target.Node.Style = lhsNode.Style
+		return addScalars(target, lhsNode, rhs.Node)
+	}
+
+	return target, nil
+}
+
+func addScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*CandidateNode, error) {
+
+	if lhs.Tag == "!!str" {
+		target.Node.Tag = "!!str"
+		target.Node.Value = lhs.Value + rhs.Value
+	} else if lhs.Tag == "!!int" && rhs.Tag == "!!int" {
+		lhsNum, err := strconv.Atoi(lhs.Value)
+		if err != nil {
+			return nil, err
+		}
+		rhsNum, err := strconv.Atoi(rhs.Value)
+		if err != nil {
+			return nil, err
+		}
+		sum := lhsNum + rhsNum
+		target.Node.Tag = "!!int"
+		target.Node.Value = fmt.Sprintf("%v", sum)
+	} else if (lhs.Tag == "!!int" || lhs.Tag == "!!float") && (rhs.Tag == "!!int" || rhs.Tag == "!!float") {
+		lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
+		if err != nil {
+			return nil, err
+		}
+		rhsNum, err := strconv.ParseFloat(rhs.Value, 64)
+		if err != nil {
+			return nil, err
+		}
+		sum := lhsNum + rhsNum
+		target.Node.Tag = "!!float"
+		target.Node.Value = fmt.Sprintf("%v", sum)
+	} else {
+		return nil, fmt.Errorf("%v cannot be added to %v", lhs.Tag, rhs.Tag)
 	}
 
 	return target, nil

pkg/yqlib/operator_add_test.go

@@ -38,11 +38,11 @@ var addOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Add object to array",
-		document:    `{a: [1,2], c: {cat: meow}}`,
-		expression:  `.a + .c`,
+		description: "Add new object to array",
+		document:    `a: [{dog: woof}]`,
+		expression:  `.a + {"cat": "meow"}`,
 		expected: []string{
-			"D0, P[a], (!!seq)::[1, 2, {cat: meow}]\n",
+			"D0, P[a], (!!seq)::[{dog: woof}, {cat: meow}]\n",
 		},
 	},
 	{
@@ -61,6 +61,48 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: [1, 2, 3, 4], b: [3, 4]}\n",
 		},
 	},
+	{
+		description: "String concatenation",
+		document:    `{a: cat, b: meow}`,
+		expression:  `.a = .a + .b`,
+		expected: []string{
+			"D0, P[], (doc)::{a: catmeow, b: meow}\n",
+		},
+	},
+	{
+		description: "Relative string concatenation",
+		document:    `{a: cat, b: meow}`,
+		expression:  `.a += .b`,
+		expected: []string{
+			"D0, P[], (doc)::{a: catmeow, b: meow}\n",
+		},
+	},
+	{
+		description:    "Number addition - float",
+		subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.",
+		document:       `{a: 3, b: 4.9}`,
+		expression:     `.a = .a + .b`,
+		expected: []string{
+			"D0, P[], (doc)::{a: 7.9, b: 4.9}\n",
+		},
+	},
+	{
+		description:    "Number addition - int",
+		subdescription: "If both the lhs and rhs are ints then the expression will be calculated with ints.",
+		document:       `{a: 3, b: 4}`,
+		expression:     `.a = .a + .b`,
+		expected: []string{
+			"D0, P[], (doc)::{a: 7, b: 4}\n",
+		},
+	},
+	{
+		description: "Increment number",
+		document:    `{a: 3}`,
+		expression:  `.a += 1`,
+		expected: []string{
+			"D0, P[], (doc)::{a: 4}\n",
+		},
+	},
 }
 
 func TestAddOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_alternative.go

@@ -7,14 +7,14 @@ import (
 // corssFunction no matches
 // can boolean use crossfunction
 
-func AlternativeOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func alternativeOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("-- alternative")
-	return crossFunction(d, matchingNodes, pathNode, alternativeFunc)
+	return crossFunction(d, matchingNodes, expressionNode, alternativeFunc)
 }
 
 func alternativeFunc(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-	lhs.Node = UnwrapDoc(lhs.Node)
-	rhs.Node = UnwrapDoc(rhs.Node)
+	lhs.Node = unwrapDoc(lhs.Node)
+	rhs.Node = unwrapDoc(rhs.Node)
 	log.Debugf("Alternative LHS: %v", lhs.Node.Tag)
 	log.Debugf("-          RHS: %v", rhs.Node.Tag)
 

pkg/yqlib/operator_anchors_aliases.go

@@ -6,13 +6,13 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-func AssignAliasOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func assignAliasOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 
 	log.Debugf("AssignAlias operator!")
 
 	aliasName := ""
-	if !pathNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+	if !expressionNode.Operation.UpdateAssign {
+		rhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Rhs)
 		if err != nil {
 			return nil, err
 		}
@@ -21,7 +21,7 @@ func AssignAliasOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNod
 		}
 	}
 
-	lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
+	lhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Lhs)
 
 	if err != nil {
 		return nil, err
@@ -31,8 +31,8 @@ func AssignAliasOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNod
 		candidate := el.Value.(*CandidateNode)
 		log.Debugf("Setting aliasName : %v", candidate.GetKey())
 
-		if pathNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(nodeToMap(candidate), pathNode.Rhs)
+		if expressionNode.Operation.UpdateAssign {
+			rhs, err := d.GetMatchingNodes(nodeToMap(candidate), expressionNode.Rhs)
 			if err != nil {
 				return nil, err
 			}
@@ -47,26 +47,26 @@ func AssignAliasOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNod
 	return matchingNodes, nil
 }
 
-func GetAliasOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func getAliasOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("GetAlias operator!")
 	var results = list.New()
 
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: candidate.Node.Value, Tag: "!!str"}
-		lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
-		results.PushBack(lengthCand)
+		result := candidate.CreateChild(nil, node)
+		results.PushBack(result)
 	}
 	return results, nil
 }
 
-func AssignAnchorOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func assignAnchorOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 
 	log.Debugf("AssignAnchor operator!")
 
 	anchorName := ""
-	if !pathNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+	if !expressionNode.Operation.UpdateAssign {
+		rhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Rhs)
 		if err != nil {
 			return nil, err
 		}
@@ -76,7 +76,7 @@ func AssignAnchorOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNo
 		}
 	}
 
-	lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
+	lhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Lhs)
 
 	if err != nil {
 		return nil, err
@@ -86,8 +86,8 @@ func AssignAnchorOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNo
 		candidate := el.Value.(*CandidateNode)
 		log.Debugf("Setting anchorName of : %v", candidate.GetKey())
 
-		if pathNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(nodeToMap(candidate), pathNode.Rhs)
+		if expressionNode.Operation.UpdateAssign {
+			rhs, err := d.GetMatchingNodes(nodeToMap(candidate), expressionNode.Rhs)
 			if err != nil {
 				return nil, err
 			}
@@ -102,7 +102,7 @@ func AssignAnchorOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNo
 	return matchingNodes, nil
 }
 
-func GetAnchorOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func getAnchorOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("GetAnchor operator!")
 	var results = list.New()
 
@@ -110,19 +110,19 @@ func GetAnchorOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode
 		candidate := el.Value.(*CandidateNode)
 		anchor := candidate.Node.Anchor
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: anchor, Tag: "!!str"}
-		lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
-		results.PushBack(lengthCand)
+		result := candidate.CreateChild(nil, node)
+		results.PushBack(result)
 	}
 	return results, nil
 }
 
-func ExplodeOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func explodeOperator(d *dataTreeNavigator, matchMap *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("-- ExplodeOperation")
 
 	for el := matchMap.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		rhs, err := d.GetMatchingNodes(nodeToMap(candidate), pathNode.Rhs)
+		rhs, err := d.GetMatchingNodes(nodeToMap(candidate), expressionNode.Rhs)
 
 		if err != nil {
 			return nil, err

pkg/yqlib/operator_assign.go

@@ -2,21 +2,21 @@ package yqlib
 
 import "container/list"
 
-func AssignUpdateOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
-	lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
+func assignUpdateOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
+	lhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Lhs)
 	if err != nil {
 		return nil, err
 	}
 	var rhs *list.List
-	if !pathNode.Operation.UpdateAssign {
-		rhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+	if !expressionNode.Operation.UpdateAssign {
+		rhs, err = d.GetMatchingNodes(matchingNodes, expressionNode.Rhs)
 	}
 
 	for el := lhs.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		if pathNode.Operation.UpdateAssign {
-			rhs, err = d.GetMatchingNodes(nodeToMap(candidate), pathNode.Rhs)
+		if expressionNode.Operation.UpdateAssign {
+			rhs, err = d.GetMatchingNodes(nodeToMap(candidate), expressionNode.Rhs)
 		}
 
 		if err != nil {
@@ -28,28 +28,24 @@ func AssignUpdateOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNo
 
 		if first != nil {
 			rhsCandidate := first.Value.(*CandidateNode)
-			rhsCandidate.Node = UnwrapDoc(rhsCandidate.Node)
+			rhsCandidate.Node = unwrapDoc(rhsCandidate.Node)
 			candidate.UpdateFrom(rhsCandidate)
 		}
 	}
-	// // if there was nothing given, perhaps we are creating a new yaml doc
-	// if matchingNodes.Len() == 0 {
-	// 	log.Debug("started with nothing, returning LHS, %v", lhs.Len())
-	// 	return lhs, nil
-	// }
+
 	return matchingNodes, nil
 }
 
 // does not update content or values
-func AssignAttributesOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
-	lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
+func assignAttributesOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
+	lhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Lhs)
 	if err != nil {
 		return nil, err
 	}
 	for el := lhs.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		rhs, err := d.GetMatchingNodes(nodeToMap(candidate), pathNode.Rhs)
+		rhs, err := d.GetMatchingNodes(nodeToMap(candidate), expressionNode.Rhs)
 
 		if err != nil {
 			return nil, err

pkg/yqlib/operator_booleans.go

@@ -7,7 +7,7 @@ import (
 )
 
 func isTruthy(c *CandidateNode) (bool, error) {
-	node := UnwrapDoc(c.Node)
+	node := unwrapDoc(c.Node)
 	value := true
 
 	if node.Tag == "!!null" {
@@ -27,8 +27,8 @@ type boolOp func(bool, bool) bool
 
 func performBoolOp(op boolOp) func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-		lhs.Node = UnwrapDoc(lhs.Node)
-		rhs.Node = UnwrapDoc(rhs.Node)
+		lhs.Node = unwrapDoc(lhs.Node)
+		rhs.Node = unwrapDoc(rhs.Node)
 
 		lhsTrue, errDecoding := isTruthy(lhs)
 		if errDecoding != nil {
@@ -44,23 +44,23 @@ func performBoolOp(op boolOp) func(d *dataTreeNavigator, lhs *CandidateNode, rhs
 	}
 }
 
-func OrOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func orOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("-- orOp")
-	return crossFunction(d, matchingNodes, pathNode, performBoolOp(
+	return crossFunction(d, matchingNodes, expressionNode, performBoolOp(
 		func(b1 bool, b2 bool) bool {
 			return b1 || b2
 		}))
 }
 
-func AndOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func andOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("-- AndOp")
-	return crossFunction(d, matchingNodes, pathNode, performBoolOp(
+	return crossFunction(d, matchingNodes, expressionNode, performBoolOp(
 		func(b1 bool, b2 bool) bool {
 			return b1 && b2
 		}))
 }
 
-func NotOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func notOperator(d *dataTreeNavigator, matchMap *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("-- notOperation")
 	var results = list.New()
 

pkg/yqlib/operator_collect.go

@@ -6,7 +6,7 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-func CollectOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func collectOperator(d *dataTreeNavigator, matchMap *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("-- collectOperation")
 
 	if matchMap.Len() == 0 {
@@ -18,21 +18,22 @@ func CollectOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTr
 	var results = list.New()
 
 	node := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
-
-	var document uint = 0
-	var path []interface{}
+	var collectC *CandidateNode
+	if matchMap.Front() != nil {
+		collectC = matchMap.Front().Value.(*CandidateNode).CreateChild(nil, node)
+		if len(collectC.Path) > 0 {
+			collectC.Path = collectC.Path[:len(collectC.Path)-1]
+		}
+	} else {
+		collectC = &CandidateNode{Node: node}
+	}
 
 	for el := matchMap.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		log.Debugf("Collecting %v", NodeToString(candidate))
-		if path == nil && candidate.Path != nil && len(candidate.Path) > 1 {
-			path = candidate.Path[:len(candidate.Path)-1]
-			document = candidate.Document
-		}
-		node.Content = append(node.Content, candidate.Node)
+		node.Content = append(node.Content, unwrapDoc(candidate.Node))
 	}
 
-	collectC := &CandidateNode{Node: node, Document: document, Path: path}
 	results.PushBack(collectC)
 
 	return results, nil

pkg/yqlib/operator_collect_object.go

@@ -17,7 +17,7 @@ import (
 ...
 */
 
-func CollectObjectOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func collectObjectOperator(d *dataTreeNavigator, matchMap *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("-- collectObjectOperation")
 
 	if matchMap.Len() == 0 {
@@ -35,7 +35,7 @@ func CollectObjectOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *
 	for el := matchMap.Front(); el != nil; el = el.Next() {
 		candidateNode := el.Value.(*CandidateNode)
 		for i := 0; i < len(first.Node.Content); i++ {
-			rotated[i].PushBack(createChildCandidate(candidateNode, i))
+			rotated[i].PushBack(candidateNode.CreateChild(i, candidateNode.Node.Content[i]))
 		}
 	}
 
@@ -52,15 +52,6 @@ func CollectObjectOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *
 
 }
 
-func createChildCandidate(candidate *CandidateNode, index int) *CandidateNode {
-	return &CandidateNode{
-		Document: candidate.Document,
-		Path:     candidate.CreateChildPath(index),
-		Filename: candidate.Filename,
-		Node:     candidate.Node.Content[index],
-	}
-}
-
 func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.List) (*list.List, error) {
 	if remainingMatches.Len() == 0 {
 		return aggregate, nil
@@ -68,8 +59,8 @@ func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.
 
 	candidate := remainingMatches.Remove(remainingMatches.Front()).(*CandidateNode)
 
-	splatted, err := Splat(d, nodeToMap(candidate),
-		&TraversePreferences{FollowAlias: false, IncludeMapKeys: false})
+	splatted, err := splat(d, nodeToMap(candidate),
+		&traversePreferences{FollowAlias: false, IncludeMapKeys: false})
 
 	for splatEl := splatted.Front(); splatEl != nil; splatEl = splatEl.Next() {
 		splatEl.Value.(*CandidateNode).Path = nil
@@ -96,7 +87,7 @@ func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.
 
 			newCandidate.Path = nil
 
-			newCandidate, err = multiply(&MultiplyPreferences{AppendArrays: false})(d, newCandidate, splatCandidate)
+			newCandidate, err = multiply(&multiplyPreferences{AppendArrays: false})(d, newCandidate, splatCandidate)
 			if err != nil {
 				return nil, err
 			}

pkg/yqlib/operator_collect_test.go

@@ -13,6 +13,15 @@ var collectOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[]\n",
 		},
 	},
+	{
+		skipDoc:  true,
+		document: "{a: apple}\n---\n{b: frog}",
+
+		expression: `[.]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- {a: apple}\n- {b: frog}\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   ``,

pkg/yqlib/operator_comments.go

@@ -7,27 +7,27 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-type CommentOpPreferences struct {
+type commentOpPreferences struct {
 	LineComment bool
 	HeadComment bool
 	FootComment bool
 }
 
-func AssignCommentsOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func assignCommentsOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 
 	log.Debugf("AssignComments operator!")
 
-	lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
+	lhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Lhs)
 
 	if err != nil {
 		return nil, err
 	}
 
-	preferences := pathNode.Operation.Preferences.(*CommentOpPreferences)
+	preferences := expressionNode.Operation.Preferences.(*commentOpPreferences)
 
 	comment := ""
-	if !pathNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+	if !expressionNode.Operation.UpdateAssign {
+		rhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Rhs)
 		if err != nil {
 			return nil, err
 		}
@@ -40,8 +40,8 @@ func AssignCommentsOperator(d *dataTreeNavigator, matchingNodes *list.List, path
 	for el := lhs.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		if pathNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(nodeToMap(candidate), pathNode.Rhs)
+		if expressionNode.Operation.UpdateAssign {
+			rhs, err := d.GetMatchingNodes(nodeToMap(candidate), expressionNode.Rhs)
 			if err != nil {
 				return nil, err
 			}
@@ -66,8 +66,8 @@ func AssignCommentsOperator(d *dataTreeNavigator, matchingNodes *list.List, path
 	return matchingNodes, nil
 }
 
-func GetCommentsOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
-	preferences := pathNode.Operation.Preferences.(*CommentOpPreferences)
+func getCommentsOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
+	preferences := expressionNode.Operation.Preferences.(*commentOpPreferences)
 	log.Debugf("GetComments operator!")
 	var results = list.New()
 
@@ -84,8 +84,8 @@ func GetCommentsOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNod
 		comment = strings.Replace(comment, "# ", "", 1)
 
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: comment, Tag: "!!str"}
-		lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
-		results.PushBack(lengthCand)
+		result := candidate.CreateChild(nil, node)
+		results.PushBack(result)
 	}
 	return results, nil
 }

pkg/yqlib/operator_create_map.go

@@ -6,7 +6,7 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-func CreateMapOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func createMapOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("-- createMapOperation")
 
 	//each matchingNodes entry should turn into a sequence of keys to create.
@@ -22,14 +22,14 @@ func CreateMapOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode
 
 		for matchingNodeEl := matchingNodes.Front(); matchingNodeEl != nil; matchingNodeEl = matchingNodeEl.Next() {
 			matchingNode := matchingNodeEl.Value.(*CandidateNode)
-			sequenceNode, err := sequenceFor(d, matchingNode, pathNode)
+			sequenceNode, err := sequenceFor(d, matchingNode, expressionNode)
 			if err != nil {
 				return nil, err
 			}
 			sequences.PushBack(sequenceNode)
 		}
 	} else {
-		sequenceNode, err := sequenceFor(d, nil, pathNode)
+		sequenceNode, err := sequenceFor(d, nil, expressionNode)
 		if err != nil {
 			return nil, err
 		}
@@ -40,7 +40,7 @@ func CreateMapOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode
 
 }
 
-func sequenceFor(d *dataTreeNavigator, matchingNode *CandidateNode, pathNode *PathTreeNode) (*CandidateNode, error) {
+func sequenceFor(d *dataTreeNavigator, matchingNode *CandidateNode, expressionNode *ExpressionNode) (*CandidateNode, error) {
 	var path []interface{}
 	var document uint = 0
 	var matches = list.New()
@@ -51,14 +51,14 @@ func sequenceFor(d *dataTreeNavigator, matchingNode *CandidateNode, pathNode *Pa
 		matches = nodeToMap(matchingNode)
 	}
 
-	mapPairs, err := crossFunction(d, matches, pathNode,
+	mapPairs, err := crossFunction(d, matches, expressionNode,
 		func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 			node := yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
 			log.Debugf("LHS:", NodeToString(lhs))
 			log.Debugf("RHS:", NodeToString(rhs))
 			node.Content = []*yaml.Node{
-				UnwrapDoc(lhs.Node),
-				UnwrapDoc(rhs.Node),
+				unwrapDoc(lhs.Node),
+				unwrapDoc(rhs.Node),
 			}
 
 			return &CandidateNode{Node: &node, Document: document, Path: path}, nil

pkg/yqlib/operator_delete.go

@@ -7,9 +7,9 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-func DeleteChildOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func deleteChildOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 
-	nodesToDelete, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+	nodesToDelete, err := d.GetMatchingNodes(matchingNodes, expressionNode.Rhs)
 
 	if err != nil {
 		return nil, err
@@ -19,11 +19,11 @@ func DeleteChildOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNod
 		candidate := el.Value.(*CandidateNode)
 
 		deleteImmediateChildOp := &Operation{
-			OperationType: DeleteImmediateChild,
+			OperationType: deleteImmediateChildOpType,
 			Value:         candidate.Path[len(candidate.Path)-1],
 		}
 
-		deleteImmediateChildOpNode := &PathTreeNode{
+		deleteImmediateChildOpNode := &ExpressionNode{
 			Operation: deleteImmediateChildOp,
 			Rhs:       createTraversalTree(candidate.Path[0 : len(candidate.Path)-1]),
 		}
@@ -36,20 +36,20 @@ func DeleteChildOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNod
 	return matchingNodes, nil
 }
 
-func DeleteImmediateChildOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
-	parents, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+func deleteImmediateChildOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
+	parents, err := d.GetMatchingNodes(matchingNodes, expressionNode.Rhs)
 
 	if err != nil {
 		return nil, err
 	}
 
-	childPath := pathNode.Operation.Value
+	childPath := expressionNode.Operation.Value
 
 	log.Debug("childPath to remove %v", childPath)
 
 	for el := parents.Front(); el != nil; el = el.Next() {
 		parent := el.Value.(*CandidateNode)
-		parentNode := UnwrapDoc(parent.Node)
+		parentNode := unwrapDoc(parent.Node)
 		if parentNode.Kind == yaml.MappingNode {
 			deleteFromMap(parent, childPath)
 		} else if parentNode.Kind == yaml.SequenceNode {
@@ -64,7 +64,7 @@ func DeleteImmediateChildOperator(d *dataTreeNavigator, matchingNodes *list.List
 
 func deleteFromMap(candidate *CandidateNode, childPath interface{}) {
 	log.Debug("deleteFromMap")
-	node := UnwrapDoc(candidate.Node)
+	node := unwrapDoc(candidate.Node)
 	contents := node.Content
 	newContents := make([]*yaml.Node, 0)
 
@@ -72,11 +72,7 @@ func deleteFromMap(candidate *CandidateNode, childPath interface{}) {
 		key := contents[index]
 		value := contents[index+1]
 
-		childCandidate := &CandidateNode{
-			Node:     value,
-			Document: candidate.Document,
-			Path:     candidate.CreateChildPath(key.Value),
-		}
+		childCandidate := candidate.CreateChild(key.Value, value)
 
 		shouldDelete := key.Value == childPath
 
@@ -91,7 +87,7 @@ func deleteFromMap(candidate *CandidateNode, childPath interface{}) {
 
 func deleteFromArray(candidate *CandidateNode, childPath interface{}) {
 	log.Debug("deleteFromArray")
-	node := UnwrapDoc(candidate.Node)
+	node := unwrapDoc(candidate.Node)
 	contents := node.Content
 	newContents := make([]*yaml.Node, 0)
 

pkg/yqlib/operator_document_index.go

@@ -7,13 +7,13 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-func GetDocumentIndexOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func getDocumentIndexOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	var results = list.New()
 
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", candidate.Document), Tag: "!!int"}
-		scalar := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
+		scalar := candidate.CreateChild(nil, node)
 		results.PushBack(scalar)
 	}
 	return results, nil

pkg/yqlib/operator_env.go

@@ -2,23 +2,24 @@ package yqlib
 
 import (
 	"container/list"
+	"fmt"
 	"os"
 	"strings"
 
 	yaml "gopkg.in/yaml.v3"
 )
 
-type EnvOpPreferences struct {
+type envOpPreferences struct {
 	StringValue bool
 }
 
-func EnvOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
-	envName := pathNode.Operation.CandidateNode.Node.Value
+func envOperator(d *dataTreeNavigator, matchMap *list.List, expressionNode *ExpressionNode) (*list.List, error) {
+	envName := expressionNode.Operation.CandidateNode.Node.Value
 	log.Debug("EnvOperator, env name:", envName)
 
 	rawValue := os.Getenv(envName)
 
-	preferences := pathNode.Operation.Preferences.(*EnvOpPreferences)
+	preferences := expressionNode.Operation.Preferences.(*envOpPreferences)
 
 	var node *yaml.Node
 	if preferences.StringValue {
@@ -27,6 +28,8 @@ func EnvOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNo
 			Tag:   "!!str",
 			Value: rawValue,
 		}
+	} else if rawValue == "" {
+		return nil, fmt.Errorf("Value for env variable '%v' not provided in env()", envName)
 	} else {
 		var dataBucket yaml.Node
 		decoder := yaml.NewDecoder(strings.NewReader(rawValue))
@@ -35,18 +38,13 @@ func EnvOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNo
 			return nil, errorReading
 		}
 		//first node is a doc
-		node = UnwrapDoc(&dataBucket)
+		node = unwrapDoc(&dataBucket)
 	}
 	log.Debug("ENV tag", node.Tag)
 	log.Debug("ENV value", node.Value)
 	log.Debug("ENV Kind", node.Kind)
 
-	target := &CandidateNode{
-		Path:     make([]interface{}, 0),
-		Document: 0,
-		Filename: "",
-		Node:     node,
-	}
+	target := &CandidateNode{Node: node}
 
 	return nodeToMap(target), nil
 }

pkg/yqlib/operator_equals.go

@@ -4,18 +4,21 @@ import (
 	"container/list"
 )
 
-func EqualsOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func equalsOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("-- equalsOperation")
-	return crossFunction(d, matchingNodes, pathNode, isEquals)
+	return crossFunction(d, matchingNodes, expressionNode, isEquals)
 }
 
 func isEquals(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	value := false
 
-	if lhs.Node.Tag == "!!null" {
-		value = (rhs.Node.Tag == "!!null")
+	lhsNode := unwrapDoc(lhs.Node)
+	rhsNode := unwrapDoc(rhs.Node)
+
+	if lhsNode.Tag == "!!null" {
+		value = (rhsNode.Tag == "!!null")
 	} else {
-		value = Match(lhs.Node.Value, rhs.Node.Value)
+		value = matchKey(lhsNode.Value, rhsNode.Value)
 	}
 	log.Debugf("%v == %v ? %v", NodeToString(lhs), NodeToString(rhs), value)
 	return createBooleanCandidate(lhs, value), nil

pkg/yqlib/operator_equals_test.go

@@ -5,6 +5,15 @@ import (
 )
 
 var equalsOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   "cat",
+		document2:  "dog",
+		expression: "select(fi==0) == select(fi==1)",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
 	{
 		description: "Match string",
 		document:    `[cat,goat,dog]`,

pkg/yqlib/operator_file.go

@@ -7,7 +7,7 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-func GetFilenameOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func getFilenameOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("GetFilename")
 
 	var results = list.New()
@@ -15,14 +15,14 @@ func GetFilenameOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNod
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: candidate.Filename, Tag: "!!str"}
-		lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
-		results.PushBack(lengthCand)
+		result := candidate.CreateChild(nil, node)
+		results.PushBack(result)
 	}
 
 	return results, nil
 }
 
-func GetFileIndexOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func getFileIndexOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("GetFileIndex")
 
 	var results = list.New()
@@ -30,8 +30,8 @@ func GetFileIndexOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNo
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", candidate.FileIndex), Tag: "!!int"}
-		lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
-		results.PushBack(lengthCand)
+		result := candidate.CreateChild(nil, node)
+		results.PushBack(result)
 	}
 
 	return results, nil

pkg/yqlib/operator_file_test.go

@@ -29,6 +29,14 @@ var fileOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!int)::0\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "a: cat\nb: dog",
+		expression: `.. lineComment |= filename`,
+		expected: []string{
+			"D0, P[], (!!map)::a: cat # sample.yml\nb: dog # sample.yml\n",
+		},
+	},
 }
 
 func TestFileOperatorsScenarios(t *testing.T) {

pkg/yqlib/operator_has.go

@@ -7,12 +7,12 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-func HasOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func hasOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 
 	log.Debugf("-- hasOperation")
 	var results = list.New()
 
-	rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+	rhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Rhs)
 	wanted := rhs.Front().Value.(*CandidateNode).Node
 	wantedKey := wanted.Value
 
@@ -24,8 +24,9 @@ func HasOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathT
 		candidate := el.Value.(*CandidateNode)
 
 		// grab the first value
-		var contents = candidate.Node.Content
-		switch candidate.Node.Kind {
+		candidateNode := unwrapDoc(candidate.Node)
+		var contents = candidateNode.Content
+		switch candidateNode.Kind {
 		case yaml.MappingNode:
 			candidateHasKey := false
 			for index := 0; index < len(contents) && !candidateHasKey; index = index + 2 {

pkg/yqlib/operator_has_test.go

@@ -5,6 +5,14 @@ import (
 )
 
 var hasOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `a: hello`,
+		expression: `has("a")`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
 	{
 		description: "Has map key",
 		document: `- a: "yes"

pkg/yqlib/operator_length.go

@@ -7,13 +7,13 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-func LengthOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func lengthOperator(d *dataTreeNavigator, matchMap *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("-- lengthOperation")
 	var results = list.New()
 
 	for el := matchMap.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		targetNode := UnwrapDoc(candidate.Node)
+		targetNode := unwrapDoc(candidate.Node)
 		var length int
 		switch targetNode.Kind {
 		case yaml.ScalarNode:
@@ -27,8 +27,8 @@ func LengthOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTre
 		}
 
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", length), Tag: "!!int"}
-		lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
-		results.PushBack(lengthCand)
+		result := candidate.CreateChild(nil, node)
+		results.PushBack(result)
 	}
 
 	return results, nil

pkg/yqlib/operator_multiply.go

@@ -8,16 +8,16 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-type CrossFunctionCalculation func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)
+type crossFunctionCalculation func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)
 
-func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode, calculation CrossFunctionCalculation) (*list.List, error) {
-	lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
+func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (*list.List, error) {
+	lhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Lhs)
 	if err != nil {
 		return nil, err
 	}
 	log.Debugf("crossFunction LHS len: %v", lhs.Len())
 
-	rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+	rhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Rhs)
 
 	if err != nil {
 		return nil, err
@@ -43,19 +43,19 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, pathNode *Pat
 	return results, nil
 }
 
-type MultiplyPreferences struct {
+type multiplyPreferences struct {
 	AppendArrays bool
 }
 
-func MultiplyOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func multiplyOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("-- MultiplyOperator")
-	return crossFunction(d, matchingNodes, pathNode, multiply(pathNode.Operation.Preferences.(*MultiplyPreferences)))
+	return crossFunction(d, matchingNodes, expressionNode, multiply(expressionNode.Operation.Preferences.(*multiplyPreferences)))
 }
 
-func multiply(preferences *MultiplyPreferences) func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+func multiply(preferences *multiplyPreferences) func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-		lhs.Node = UnwrapDoc(lhs.Node)
-		rhs.Node = UnwrapDoc(rhs.Node)
+		lhs.Node = unwrapDoc(lhs.Node)
+		rhs.Node = unwrapDoc(rhs.Node)
 		log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
 		log.Debugf("-          RHS: %v", rhs.Node.Tag)
 
@@ -64,12 +64,7 @@ func multiply(preferences *MultiplyPreferences) func(d *dataTreeNavigator, lhs *
 		if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
 			(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
 
-			var newBlank = &CandidateNode{
-				Path:     lhs.Path,
-				Document: lhs.Document,
-				Filename: lhs.Filename,
-				Node:     &yaml.Node{},
-			}
+			var newBlank = lhs.CreateChild(nil, &yaml.Node{})
 			var newThing, err = mergeObjects(d, newBlank, lhs, false)
 			if err != nil {
 				return nil, err
@@ -85,8 +80,8 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode,
 	var results = list.New()
 
 	// shouldn't recurse arrays if appending
-	prefs := &RecursiveDescentPreferences{RecurseArray: !shouldAppendArrays,
-		TraversePreferences: &TraversePreferences{FollowAlias: false}}
+	prefs := &recursiveDescentPreferences{RecurseArray: !shouldAppendArrays,
+		TraversePreferences: &traversePreferences{FollowAlias: false}}
 	err := recursiveDecent(d, results, nodeToMap(rhs), prefs)
 	if err != nil {
 		return nil, err
@@ -112,16 +107,16 @@ func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *Candid
 
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
 
-	assignmentOp := &Operation{OperationType: AssignAttributes}
+	assignmentOp := &Operation{OperationType: assignAttributesOpType}
 	if rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode {
-		assignmentOp.OperationType = Assign
+		assignmentOp.OperationType = assignOpType
 		assignmentOp.UpdateAssign = false
 	} else if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
-		assignmentOp.OperationType = AddAssign
+		assignmentOp.OperationType = addAssignOpType
 	}
-	rhsOp := &Operation{OperationType: ValueOp, CandidateNode: rhs}
+	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
 
-	assignmentOpNode := &PathTreeNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath), Rhs: &PathTreeNode{Operation: rhsOp}}
+	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath), Rhs: &ExpressionNode{Operation: rhsOp}}
 
 	_, err := d.GetMatchingNodes(nodeToMap(lhs), assignmentOpNode)
 

pkg/yqlib/operator_multiply_test.go

@@ -132,11 +132,11 @@ b:
 		},
 	},
 	{
-		description: "Merge does not copy anchor names",
+		description: "Merge copies anchor names",
 		document:    `{a: {c: &cat frog}, b: {f: *cat}, c: {g: thongs}}`,
 		expression:  `.c * .a`,
 		expected: []string{
-			"D0, P[c], (!!map)::{g: thongs, c: frog}\n",
+			"D0, P[c], (!!map)::{g: thongs, c: &cat frog}\n",
 		},
 	},
 	{

pkg/yqlib/operator_path.go

@@ -16,7 +16,7 @@ func createPathNodeFor(pathElement interface{}) *yaml.Node {
 	}
 }
 
-func GetPathOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func getPathOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("GetPath")
 
 	var results = list.New()
@@ -31,8 +31,8 @@ func GetPathOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *P
 			content[pathIndex] = createPathNodeFor(path)
 		}
 		node.Content = content
-		lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
-		results.PushBack(lengthCand)
+		result := candidate.CreateChild(nil, node)
+		results.PushBack(result)
 	}
 
 	return results, nil

pkg/yqlib/operator_pipe.go

@@ -2,10 +2,10 @@ package yqlib
 
 import "container/list"
 
-func PipeOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
-	lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
+func pipeOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
+	lhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Lhs)
 	if err != nil {
 		return nil, err
 	}
-	return d.GetMatchingNodes(lhs, pathNode.Rhs)
+	return d.GetMatchingNodes(lhs, expressionNode.Rhs)
 }

pkg/yqlib/operator_recursive_descent.go

@@ -6,15 +6,15 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-type RecursiveDescentPreferences struct {
-	TraversePreferences *TraversePreferences
+type recursiveDescentPreferences struct {
+	TraversePreferences *traversePreferences
 	RecurseArray        bool
 }
 
-func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func recursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	var results = list.New()
 
-	preferences := pathNode.Operation.Preferences.(*RecursiveDescentPreferences)
+	preferences := expressionNode.Operation.Preferences.(*recursiveDescentPreferences)
 	err := recursiveDecent(d, results, matchMap, preferences)
 	if err != nil {
 		return nil, err
@@ -23,11 +23,11 @@ func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNod
 	return results, nil
 }
 
-func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List, preferences *RecursiveDescentPreferences) error {
+func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List, preferences *recursiveDescentPreferences) error {
 	for el := matchMap.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		candidate.Node = UnwrapDoc(candidate.Node)
+		candidate.Node = unwrapDoc(candidate.Node)
 
 		log.Debugf("Recursive Decent, added %v", NodeToString(candidate))
 		results.PushBack(candidate)
@@ -35,7 +35,7 @@ func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.Li
 		if candidate.Node.Kind != yaml.AliasNode && len(candidate.Node.Content) > 0 &&
 			(preferences.RecurseArray || candidate.Node.Kind != yaml.SequenceNode) {
 
-			children, err := Splat(d, nodeToMap(candidate), preferences.TraversePreferences)
+			children, err := splat(d, nodeToMap(candidate), preferences.TraversePreferences)
 
 			if err != nil {
 				return err

pkg/yqlib/operator_recursive_descent_test.go

@@ -63,12 +63,12 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Recursively find nodes with keys",
-		document:    `{a: {name: frog, b: {name: blog, age: 12}}}`,
-		expression:  `.. | select(has("name"))`,
+		description:    "Recursively find nodes with keys",
+		subdescription: "Note that this example has wrapped the expression in `[]` to show that there are two matches returned. You do not have to wrap in `[]` in your path expression.",
+		document:       `{a: {name: frog, b: {name: blog, age: 12}}}`,
+		expression:     `[.. | select(has("name"))]`,
 		expected: []string{
-			"D0, P[a], (!!map)::{name: frog, b: {name: blog, age: 12}}\n",
-			"D0, P[a b], (!!map)::{name: blog, age: 12}\n",
+			"D0, P[], (!!seq)::- {name: frog, b: {name: blog, age: 12}}\n- {name: blog, age: 12}\n",
 		},
 	},
 	{
@@ -165,7 +165,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
 		document:    `{a: &cat {c: frog}, b: *cat}`,
 		expression:  `[..]`,
 		expected: []string{
-			"D0, P[a], (!!seq)::- {a: &cat {c: frog}, b: *cat}\n- &cat {c: frog}\n- frog\n- *cat\n",
+			"D0, P[], (!!seq)::- {a: &cat {c: frog}, b: *cat}\n- &cat {c: frog}\n- frog\n- *cat\n",
 		},
 	},
 	{
@@ -187,7 +187,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
 		document:    mergeDocSample,
 		expression:  `.foobar | [..]`,
 		expected: []string{
-			"D0, P[foobar], (!!seq)::- c: foobar_c\n  !!merge <<: *foo\n  thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n",
+			"D0, P[], (!!seq)::- c: foobar_c\n  !!merge <<: *foo\n  thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n",
 		},
 	},
 	{
@@ -195,7 +195,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
 		document:   mergeDocSample,
 		expression: `.foobar | [...]`,
 		expected: []string{
-			"D0, P[foobar], (!!seq)::- c: foobar_c\n  !!merge <<: *foo\n  thing: foobar_thing\n- c\n- foobar_c\n- !!merge <<\n- *foo\n- thing\n- foobar_thing\n",
+			"D0, P[], (!!seq)::- c: foobar_c\n  !!merge <<: *foo\n  thing: foobar_thing\n- c\n- foobar_c\n- !!merge <<\n- *foo\n- thing\n- foobar_thing\n",
 		},
 	},
 	{
@@ -233,5 +233,5 @@ func TestRecursiveDescentOperatorScenarios(t *testing.T) {
 	for _, tt := range recursiveDescentOperatorScenarios {
 		testScenario(t, &tt)
 	}
-	documentScenarios(t, "Recursive Descent", recursiveDescentOperatorScenarios)
+	documentScenarios(t, "Recursive Descent (Glob)", recursiveDescentOperatorScenarios)
 }

pkg/yqlib/operator_select.go

@@ -4,7 +4,7 @@ import (
 	"container/list"
 )
 
-func SelectOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func selectOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 
 	log.Debugf("-- selectOperation")
 	var results = list.New()
@@ -12,7 +12,7 @@ func SelectOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *Pa
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		rhs, err := d.GetMatchingNodes(nodeToMap(candidate), pathNode.Rhs)
+		rhs, err := d.GetMatchingNodes(nodeToMap(candidate), expressionNode.Rhs)
 
 		if err != nil {
 			return nil, err

pkg/yqlib/operator_self.go

@@ -2,6 +2,6 @@ package yqlib
 
 import "container/list"
 
-func SelfOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func selfOperator(d *dataTreeNavigator, matchMap *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	return matchMap, nil
 }

pkg/yqlib/operator_sort_keys.go

@@ -7,17 +7,17 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-func SortKeysOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func sortKeysOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		rhs, err := d.GetMatchingNodes(nodeToMap(candidate), pathNode.Rhs)
+		rhs, err := d.GetMatchingNodes(nodeToMap(candidate), expressionNode.Rhs)
 		if err != nil {
 			return nil, err
 		}
 
 		for childEl := rhs.Front(); childEl != nil; childEl = childEl.Next() {
-			node := UnwrapDoc(childEl.Value.(*CandidateNode).Node)
+			node := unwrapDoc(childEl.Value.(*CandidateNode).Node)
 			if node.Kind == yaml.MappingNode {
 				sortKeys(node)
 			}

pkg/yqlib/operator_style.go

@@ -26,12 +26,12 @@ func parseStyle(customStyle string) (yaml.Style, error) {
 	return 0, nil
 }
 
-func AssignStyleOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func assignStyleOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 
 	log.Debugf("AssignStyleOperator: %v")
 	var style yaml.Style
-	if !pathNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+	if !expressionNode.Operation.UpdateAssign {
+		rhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Rhs)
 		if err != nil {
 			return nil, err
 		}
@@ -44,7 +44,7 @@ func AssignStyleOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNod
 		}
 	}
 
-	lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
+	lhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Lhs)
 
 	if err != nil {
 		return nil, err
@@ -53,8 +53,8 @@ func AssignStyleOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNod
 	for el := lhs.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		log.Debugf("Setting style of : %v", candidate.GetKey())
-		if pathNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(nodeToMap(candidate), pathNode.Rhs)
+		if expressionNode.Operation.UpdateAssign {
+			rhs, err := d.GetMatchingNodes(nodeToMap(candidate), expressionNode.Rhs)
 			if err != nil {
 				return nil, err
 			}
@@ -73,7 +73,7 @@ func AssignStyleOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNod
 	return matchingNodes, nil
 }
 
-func GetStyleOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func getStyleOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("GetStyleOperator")
 
 	var results = list.New()
@@ -100,8 +100,8 @@ func GetStyleOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *
 			style = "<unknown>"
 		}
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: style, Tag: "!!str"}
-		lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
-		results.PushBack(lengthCand)
+		result := candidate.CreateChild(nil, node)
+		results.PushBack(result)
 	}
 
 	return results, nil

pkg/yqlib/operator_tag.go

@@ -6,13 +6,13 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-func AssignTagOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func assignTagOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 
 	log.Debugf("AssignTagOperator: %v")
 	tag := ""
 
-	if !pathNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+	if !expressionNode.Operation.UpdateAssign {
+		rhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Rhs)
 		if err != nil {
 			return nil, err
 		}
@@ -22,7 +22,7 @@ func AssignTagOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode
 		}
 	}
 
-	lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
+	lhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Lhs)
 
 	if err != nil {
 		return nil, err
@@ -31,8 +31,8 @@ func AssignTagOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode
 	for el := lhs.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		log.Debugf("Setting tag of : %v", candidate.GetKey())
-		if pathNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(nodeToMap(candidate), pathNode.Rhs)
+		if expressionNode.Operation.UpdateAssign {
+			rhs, err := d.GetMatchingNodes(nodeToMap(candidate), expressionNode.Rhs)
 			if err != nil {
 				return nil, err
 			}
@@ -41,22 +41,22 @@ func AssignTagOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode
 				tag = rhs.Front().Value.(*CandidateNode).Node.Value
 			}
 		}
-		candidate.Node.Tag = tag
+		unwrapDoc(candidate.Node).Tag = tag
 	}
 
 	return matchingNodes, nil
 }
 
-func GetTagOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func getTagOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("GetTagOperator")
 
 	var results = list.New()
 
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		node := &yaml.Node{Kind: yaml.ScalarNode, Value: UnwrapDoc(candidate.Node).Tag, Tag: "!!str"}
-		lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
-		results.PushBack(lengthCand)
+		node := &yaml.Node{Kind: yaml.ScalarNode, Value: unwrapDoc(candidate.Node).Tag, Tag: "!!str"}
+		result := candidate.CreateChild(nil, node)
+		results.PushBack(result)
 	}
 
 	return results, nil

pkg/yqlib/operator_tag_test.go

@@ -26,6 +26,14 @@ var tagOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!str)::'!!map'\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `32`,
+		expression: `. tag= "!!str"`,
+		expected: []string{
+			"D0, P[], (doc)::\"32\"\n",
+		},
+	},
 	{
 		description: "Set custom tag",
 		document:    `{a: str}`,

pkg/yqlib/operator_traverse_path.go

@@ -9,21 +9,21 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-type TraversePreferences struct {
+type traversePreferences struct {
 	FollowAlias    bool
 	IncludeMapKeys bool
 }
 
-func Splat(d *dataTreeNavigator, matches *list.List, prefs *TraversePreferences) (*list.List, error) {
+func splat(d *dataTreeNavigator, matches *list.List, prefs *traversePreferences) (*list.List, error) {
 	return traverseNodesWithArrayIndices(matches, make([]*yaml.Node, 0), prefs)
 }
 
-func TraversePathOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func traversePathOperator(d *dataTreeNavigator, matchMap *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("-- Traversing")
 	var matchingNodeMap = list.New()
 
 	for el := matchMap.Front(); el != nil; el = el.Next() {
-		newNodes, err := traverse(d, el.Value.(*CandidateNode), pathNode.Operation)
+		newNodes, err := traverse(d, el.Value.(*CandidateNode), expressionNode.Operation)
 		if err != nil {
 			return nil, err
 		}
@@ -54,7 +54,7 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Oper
 	switch value.Kind {
 	case yaml.MappingNode:
 		log.Debug("its a map with %v entries", len(value.Content)/2)
-		prefs := &TraversePreferences{FollowAlias: true}
+		prefs := &traversePreferences{FollowAlias: true}
 		return traverseMap(matchingNode, operation.StringValue, prefs, false)
 
 	case yaml.SequenceNode:
@@ -67,30 +67,27 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Oper
 		return traverse(d, matchingNode, operation)
 	case yaml.DocumentNode:
 		log.Debug("digging into doc node")
-		return traverse(d, &CandidateNode{
-			Node:      matchingNode.Node.Content[0],
-			Filename:  matchingNode.Filename,
-			FileIndex: matchingNode.FileIndex,
-			Document:  matchingNode.Document}, operation)
+
+		return traverse(d, matchingNode.CreateChild(nil, matchingNode.Node.Content[0]), operation)
 	default:
 		return list.New(), nil
 	}
 }
 
-func TraverseArrayOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func traverseArrayOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	// rhs is a collect expression that will yield indexes to retreive of the arrays
 
-	rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+	rhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Rhs)
 	if err != nil {
 		return nil, err
 	}
 
 	var indicesToTraverse = rhs.Front().Value.(*CandidateNode).Node.Content
-	prefs := &TraversePreferences{FollowAlias: true}
+	prefs := &traversePreferences{FollowAlias: true}
 	return traverseNodesWithArrayIndices(matchingNodes, indicesToTraverse, prefs)
 }
 
-func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, prefs *TraversePreferences) (*list.List, error) {
+func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, prefs *traversePreferences) (*list.List, error) {
 	var matchingNodeMap = list.New()
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
@@ -104,7 +101,7 @@ func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse [
 	return matchingNodeMap, nil
 }
 
-func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, prefs *TraversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
+func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, prefs *traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
 	node := matchingNode.Node
 	if node.Tag == "!!null" {
 		log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
@@ -121,17 +118,13 @@ func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml
 	} else if node.Kind == yaml.MappingNode {
 		return traverseMapWithIndices(matchingNode, indicesToTraverse, prefs)
 	} else if node.Kind == yaml.DocumentNode {
-		return traverseArrayIndices(&CandidateNode{
-			Node:      matchingNode.Node.Content[0],
-			Filename:  matchingNode.Filename,
-			FileIndex: matchingNode.FileIndex,
-			Document:  matchingNode.Document}, indicesToTraverse, prefs)
+		return traverseArrayIndices(matchingNode.CreateChild(nil, matchingNode.Node.Content[0]), indicesToTraverse, prefs)
 	}
 	log.Debugf("OperatorArrayTraverse skipping %v as its a %v", matchingNode, node.Tag)
 	return list.New(), nil
 }
 
-func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs *TraversePreferences) (*list.List, error) {
+func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs *traversePreferences) (*list.List, error) {
 	if len(indices) == 0 {
 		return traverseMap(candidate, "", prefs, true)
 	}
@@ -153,17 +146,13 @@ func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, pref
 func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*list.List, error) {
 	log.Debug("traverseArrayWithIndices")
 	var newMatches = list.New()
-	node := UnwrapDoc(candidate.Node)
+	node := unwrapDoc(candidate.Node)
 	if len(indices) == 0 {
 		log.Debug("splatting")
 		var index int64
 		for index = 0; index < int64(len(node.Content)); index = index + 1 {
 
-			newMatches.PushBack(&CandidateNode{
-				Document: candidate.Document,
-				Path:     candidate.CreateChildPath(index),
-				Node:     node.Content[index],
-			})
+			newMatches.PushBack(candidate.CreateChild(index, node.Content[index]))
 		}
 		return newMatches, nil
 
@@ -190,20 +179,16 @@ func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*
 			return nil, fmt.Errorf("Index [%v] out of range, array size is %v", index, contentLength)
 		}
 
-		newMatches.PushBack(&CandidateNode{
-			Node:     node.Content[indexToUse],
-			Document: candidate.Document,
-			Path:     candidate.CreateChildPath(index),
-		})
+		newMatches.PushBack(candidate.CreateChild(index, node.Content[indexToUse]))
 	}
 	return newMatches, nil
 }
 
 func keyMatches(key *yaml.Node, wantedKey string) bool {
-	return Match(key.Value, wantedKey)
+	return matchKey(key.Value, wantedKey)
 }
 
-func traverseMap(matchingNode *CandidateNode, key string, prefs *TraversePreferences, splat bool) (*list.List, error) {
+func traverseMap(matchingNode *CandidateNode, key string, prefs *traversePreferences, splat bool) (*list.List, error) {
 	var newMatches = orderedmap.NewOrderedMap()
 	err := doTraverseMap(newMatches, matchingNode, key, prefs, splat)
 
@@ -216,13 +201,8 @@ func traverseMap(matchingNode *CandidateNode, key string, prefs *TraversePrefere
 		valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}
 		node := matchingNode.Node
 		node.Content = append(node.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: key}, valueNode)
-		candidateNode := &CandidateNode{
-			Node:     valueNode,
-			Path:     append(matchingNode.Path, key),
-			Document: matchingNode.Document,
-		}
+		candidateNode := matchingNode.CreateChild(key, valueNode)
 		newMatches.Set(candidateNode.GetKey(), candidateNode)
-
 	}
 
 	results := list.New()
@@ -234,7 +214,7 @@ func traverseMap(matchingNode *CandidateNode, key string, prefs *TraversePrefere
 	return results, nil
 }
 
-func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, prefs *TraversePreferences, splat bool) error {
+func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, prefs *traversePreferences, splat bool) error {
 	// value.Content is a concatenated array of key, value,
 	// so keys are in the even indexes, values in odd.
 	// merge aliases are defined first, but we only want to traverse them
@@ -258,18 +238,10 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode,
 		} else if splat || keyMatches(key, wantedKey) {
 			log.Debug("MATCHED")
 			if prefs.IncludeMapKeys {
-				candidateNode := &CandidateNode{
-					Node:     key,
-					Path:     candidate.CreateChildPath(key.Value),
-					Document: candidate.Document,
-				}
+				candidateNode := candidate.CreateChild(key.Value, key)
 				newMatches.Set(fmt.Sprintf("keyOf-%v", candidateNode.GetKey()), candidateNode)
 			}
-			candidateNode := &CandidateNode{
-				Node:     value,
-				Path:     candidate.CreateChildPath(key.Value),
-				Document: candidate.Document,
-			}
+			candidateNode := candidate.CreateChild(key.Value, value)
 			newMatches.Set(candidateNode.GetKey(), candidateNode)
 		}
 	}
@@ -277,14 +249,10 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode,
 	return nil
 }
 
-func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, prefs *TraversePreferences, splat bool) error {
+func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, prefs *traversePreferences, splat bool) error {
 	switch value.Kind {
 	case yaml.AliasNode:
-		candidateNode := &CandidateNode{
-			Node:     value.Alias,
-			Path:     originalCandidate.Path,
-			Document: originalCandidate.Document,
-		}
+		candidateNode := originalCandidate.CreateChild(nil, value.Alias)
 		return doTraverseMap(newMatches, candidateNode, wantedKey, prefs, splat)
 	case yaml.SequenceNode:
 		for _, childValue := range value.Content {

pkg/yqlib/operator_union.go

@@ -2,12 +2,12 @@ package yqlib
 
 import "container/list"
 
-func UnionOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
-	lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
+func unionOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
+	lhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Lhs)
 	if err != nil {
 		return nil, err
 	}
-	rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+	rhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Rhs)
 	if err != nil {
 		return nil, err
 	}

pkg/yqlib/operator_value.go

@@ -2,7 +2,7 @@ package yqlib
 
 import "container/list"
 
-func ValueOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
-	log.Debug("value = %v", pathNode.Operation.CandidateNode.Node.Value)
-	return nodeToMap(pathNode.Operation.CandidateNode), nil
+func valueOperator(d *dataTreeNavigator, matchMap *list.List, expressionNode *ExpressionNode) (*list.List, error) {
+	log.Debug("value = %v", expressionNode.Operation.CandidateNode.Node.Value)
+	return nodeToMap(expressionNode.Operation.CandidateNode), nil
 }

pkg/yqlib/operators.go

@@ -7,16 +7,16 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-type OperatorHandler func(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error)
+type operatorHandler func(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error)
 
-func UnwrapDoc(node *yaml.Node) *yaml.Node {
+func unwrapDoc(node *yaml.Node) *yaml.Node {
 	if node.Kind == yaml.DocumentNode {
 		return node.Content[0]
 	}
 	return node
 }
 
-func EmptyOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+func emptyOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	return list.New(), nil
 }
 
@@ -26,7 +26,7 @@ func createBooleanCandidate(owner *CandidateNode, value bool) *CandidateNode {
 		valString = "false"
 	}
 	node := &yaml.Node{Kind: yaml.ScalarNode, Value: valString, Tag: "!!bool"}
-	return &CandidateNode{Node: node, Document: owner.Document, Path: owner.Path}
+	return owner.CreateChild(nil, node)
 }
 
 func nodeToMap(candidate *CandidateNode) *list.List {
@@ -35,14 +35,14 @@ func nodeToMap(candidate *CandidateNode) *list.List {
 	return elMap
 }
 
-func createTraversalTree(path []interface{}) *PathTreeNode {
+func createTraversalTree(path []interface{}) *ExpressionNode {
 	if len(path) == 0 {
-		return &PathTreeNode{Operation: &Operation{OperationType: SelfReference}}
+		return &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
 	} else if len(path) == 1 {
-		return &PathTreeNode{Operation: &Operation{OperationType: TraversePath, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
+		return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
 	}
-	return &PathTreeNode{
-		Operation: &Operation{OperationType: ShortPipe},
+	return &ExpressionNode{
+		Operation: &Operation{OperationType: shortPipeOpType},
 		Lhs:       createTraversalTree(path[0:1]),
 		Rhs:       createTraversalTree(path[1:])}
 }

pkg/yqlib/operators_test.go

@@ -30,7 +30,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 	var results *list.List
 	var err error
 
-	node, err := treeCreator.ParsePath(s.expression)
+	node, err := NewExpressionParser().ParseExpression(s.expression)
 	if err != nil {
 		t.Error(fmt.Errorf("Error parsing expression %v of %v: %v", s.expression, s.description, err))
 		return
@@ -66,7 +66,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 		os.Setenv("myenv", s.environmentVariable)
 	}
 
-	results, err = treeNavigator.GetMatchingNodes(inputs, node)
+	results, err = NewDataTreeNavigator().GetMatchingNodes(inputs, node)
 
 	if err != nil {
 		t.Error(fmt.Errorf("%v: %v", err, s.expression))
@@ -110,7 +110,7 @@ func formatYaml(yaml string, filename string) string {
 	var output bytes.Buffer
 	printer := NewPrinter(bufio.NewWriter(&output), false, true, false, 2, true)
 
-	node, err := treeCreator.ParsePath(".. style= \"\"")
+	node, err := NewExpressionParser().ParseExpression(".. style= \"\"")
 	if err != nil {
 		panic(err)
 	}
@@ -219,7 +219,7 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt
 	var err error
 	printer := NewPrinter(bufio.NewWriter(&output), false, true, false, 2, true)
 
-	node, err := treeCreator.ParsePath(s.expression)
+	node, err := NewExpressionParser().ParseExpression(s.expression)
 	if err != nil {
 		t.Error(fmt.Errorf("Error parsing expression %v of %v: %v", s.expression, s.description, err))
 		return
@@ -252,7 +252,7 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt
 
 	}
 
-	results, err := treeNavigator.GetMatchingNodes(inputs, node)
+	results, err := NewDataTreeNavigator().GetMatchingNodes(inputs, node)
 	if err != nil {
 		t.Error(err, s.expression)
 	}

pkg/yqlib/path_tokeniser.go

@@ -1,407 +0,0 @@
-package yqlib
-
-import (
-	"fmt"
-	"strconv"
-
-	lex "github.com/timtadh/lexmachine"
-	"github.com/timtadh/lexmachine/machines"
-)
-
-func skip(*lex.Scanner, *machines.Match) (interface{}, error) {
-	return nil, nil
-}
-
-type TokenType uint32
-
-const (
-	OperationToken = 1 << iota
-	OpenBracket
-	CloseBracket
-	OpenCollect
-	CloseCollect
-	OpenCollectObject
-	CloseCollectObject
-	TraverseArrayCollect
-)
-
-type Token struct {
-	TokenType            TokenType
-	Operation            *Operation
-	AssignOperation      *Operation // e.g. tag (GetTag) op becomes AssignTag if '=' follows it
-	CheckForPostTraverse bool       // e.g. [1]cat should really be [1].cat
-
-}
-
-func (t *Token) toString() string {
-	if t.TokenType == OperationToken {
-		log.Debug("toString, its an op")
-		return t.Operation.toString()
-	} else if t.TokenType == OpenBracket {
-		return "("
-	} else if t.TokenType == CloseBracket {
-		return ")"
-	} else if t.TokenType == OpenCollect {
-		return "["
-	} else if t.TokenType == CloseCollect {
-		return "]"
-	} else if t.TokenType == OpenCollectObject {
-		return "{"
-	} else if t.TokenType == CloseCollectObject {
-		return "}"
-	} else if t.TokenType == TraverseArrayCollect {
-		return ".["
-
-	} else {
-		return "NFI"
-	}
-}
-
-func pathToken(wrapped bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		value = value[1:]
-		if wrapped {
-			value = unwrap(value)
-		}
-		log.Debug("PathToken %v", value)
-		op := &Operation{OperationType: TraversePath, Value: value, StringValue: value}
-		return &Token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil
-	}
-}
-
-func documentToken() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		var numberString = string(m.Bytes)
-		numberString = numberString[1:]
-		var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) // nolint
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-		log.Debug("documentToken %v", string(m.Bytes))
-		op := &Operation{OperationType: DocumentFilter, Value: number, StringValue: numberString}
-		return &Token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil
-	}
-}
-
-func opToken(op *OperationType) lex.Action {
-	return opTokenWithPrefs(op, nil, nil)
-}
-
-func opAssignableToken(opType *OperationType, assignOpType *OperationType) lex.Action {
-	return opTokenWithPrefs(opType, assignOpType, nil)
-}
-
-func assignOpToken(updateAssign bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		log.Debug("assignOpToken %v", string(m.Bytes))
-		value := string(m.Bytes)
-		op := &Operation{OperationType: Assign, Value: Assign.Type, StringValue: value, UpdateAssign: updateAssign}
-		return &Token{TokenType: OperationToken, Operation: op}, nil
-	}
-}
-
-func opTokenWithPrefs(op *OperationType, assignOpType *OperationType, preferences interface{}) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		log.Debug("opTokenWithPrefs %v", string(m.Bytes))
-		value := string(m.Bytes)
-		op := &Operation{OperationType: op, Value: op.Type, StringValue: value, Preferences: preferences}
-		var assign *Operation
-		if assignOpType != nil {
-			assign = &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, Preferences: preferences}
-		}
-		return &Token{TokenType: OperationToken, Operation: op, AssignOperation: assign}, nil
-	}
-}
-
-func assignAllCommentsOp(updateAssign bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		log.Debug("assignAllCommentsOp %v", string(m.Bytes))
-		value := string(m.Bytes)
-		op := &Operation{
-			OperationType: AssignComment,
-			Value:         AssignComment.Type,
-			StringValue:   value,
-			UpdateAssign:  updateAssign,
-			Preferences:   &CommentOpPreferences{LineComment: true, HeadComment: true, FootComment: true},
-		}
-		return &Token{TokenType: OperationToken, Operation: op}, nil
-	}
-}
-
-func literalToken(pType TokenType, checkForPost bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &Token{TokenType: pType, CheckForPostTraverse: checkForPost}, nil
-	}
-}
-
-func unwrap(value string) string {
-	return value[1 : len(value)-1]
-}
-
-func numberValue() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		var numberString = string(m.Bytes)
-		var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) // nolint
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-
-		return &Token{TokenType: OperationToken, Operation: CreateValueOperation(number, numberString)}, nil
-	}
-}
-
-func floatValue() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		var numberString = string(m.Bytes)
-		var number, errParsingInt = strconv.ParseFloat(numberString, 64) // nolint
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-		return &Token{TokenType: OperationToken, Operation: CreateValueOperation(number, numberString)}, nil
-	}
-}
-
-func booleanValue(val bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &Token{TokenType: OperationToken, Operation: CreateValueOperation(val, string(m.Bytes))}, nil
-	}
-}
-
-func stringValue(wrapped bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		if wrapped {
-			value = unwrap(value)
-		}
-		return &Token{TokenType: OperationToken, Operation: CreateValueOperation(value, value)}, nil
-	}
-}
-
-func envOp(strenv bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		preferences := &EnvOpPreferences{}
-
-		if strenv {
-			// strenv( )
-			value = value[7 : len(value)-1]
-			preferences.StringValue = true
-		} else {
-			//env( )
-			value = value[4 : len(value)-1]
-		}
-
-		envOperation := CreateValueOperation(value, value)
-		envOperation.OperationType = EnvOp
-		envOperation.Preferences = preferences
-
-		return &Token{TokenType: OperationToken, Operation: envOperation}, nil
-	}
-}
-
-func nullValue() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &Token{TokenType: OperationToken, Operation: CreateValueOperation(nil, string(m.Bytes))}, nil
-	}
-}
-
-func selfToken() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		op := &Operation{OperationType: SelfReference}
-		return &Token{TokenType: OperationToken, Operation: op}, nil
-	}
-}
-
-func initLexer() (*lex.Lexer, error) {
-	lexer := lex.NewLexer()
-	lexer.Add([]byte(`\(`), literalToken(OpenBracket, false))
-	lexer.Add([]byte(`\)`), literalToken(CloseBracket, true))
-
-	lexer.Add([]byte(`\.\[`), literalToken(TraverseArrayCollect, false))
-	lexer.Add([]byte(`\.\.`), opTokenWithPrefs(RecursiveDescent, nil, &RecursiveDescentPreferences{RecurseArray: true,
-		TraversePreferences: &TraversePreferences{FollowAlias: false, IncludeMapKeys: false}}))
-
-	lexer.Add([]byte(`\.\.\.`), opTokenWithPrefs(RecursiveDescent, nil, &RecursiveDescentPreferences{RecurseArray: true,
-		TraversePreferences: &TraversePreferences{FollowAlias: false, IncludeMapKeys: true}}))
-
-	lexer.Add([]byte(`,`), opToken(Union))
-	lexer.Add([]byte(`:\s*`), opToken(CreateMap))
-	lexer.Add([]byte(`length`), opToken(Length))
-	lexer.Add([]byte(`sortKeys`), opToken(SortKeys))
-	lexer.Add([]byte(`select`), opToken(Select))
-	lexer.Add([]byte(`has`), opToken(Has))
-	lexer.Add([]byte(`explode`), opToken(Explode))
-	lexer.Add([]byte(`or`), opToken(Or))
-	lexer.Add([]byte(`and`), opToken(And))
-	lexer.Add([]byte(`not`), opToken(Not))
-	lexer.Add([]byte(`\/\/`), opToken(Alternative))
-
-	lexer.Add([]byte(`documentIndex`), opToken(GetDocumentIndex))
-	lexer.Add([]byte(`di`), opToken(GetDocumentIndex))
-
-	lexer.Add([]byte(`style`), opAssignableToken(GetStyle, AssignStyle))
-
-	lexer.Add([]byte(`tag`), opAssignableToken(GetTag, AssignTag))
-	lexer.Add([]byte(`anchor`), opAssignableToken(GetAnchor, AssignAnchor))
-	lexer.Add([]byte(`alias`), opAssignableToken(GetAlias, AssignAlias))
-	lexer.Add([]byte(`filename`), opToken(GetFilename))
-	lexer.Add([]byte(`fileIndex`), opToken(GetFileIndex))
-	lexer.Add([]byte(`fi`), opToken(GetFileIndex))
-	lexer.Add([]byte(`path`), opToken(GetPath))
-
-	lexer.Add([]byte(`lineComment`), opTokenWithPrefs(GetComment, AssignComment, &CommentOpPreferences{LineComment: true}))
-
-	lexer.Add([]byte(`headComment`), opTokenWithPrefs(GetComment, AssignComment, &CommentOpPreferences{HeadComment: true}))
-
-	lexer.Add([]byte(`footComment`), opTokenWithPrefs(GetComment, AssignComment, &CommentOpPreferences{FootComment: true}))
-
-	lexer.Add([]byte(`comments\s*=`), assignAllCommentsOp(false))
-	lexer.Add([]byte(`comments\s*\|=`), assignAllCommentsOp(true))
-
-	lexer.Add([]byte(`collect`), opToken(Collect))
-
-	lexer.Add([]byte(`\s*==\s*`), opToken(Equals))
-	lexer.Add([]byte(`\s*=\s*`), assignOpToken(false))
-
-	lexer.Add([]byte(`del`), opToken(DeleteChild))
-
-	lexer.Add([]byte(`\s*\|=\s*`), assignOpToken(true))
-
-	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
-
-	lexer.Add([]byte(`d[0-9]+`), documentToken())
-	lexer.Add([]byte(`\."[^ "]+"`), pathToken(true))
-	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+`), pathToken(false))
-	lexer.Add([]byte(`\.`), selfToken())
-
-	lexer.Add([]byte(`\|`), opToken(Pipe))
-
-	lexer.Add([]byte(`-?\d+(\.\d+)`), floatValue())
-	lexer.Add([]byte(`-?[1-9](\.\d+)?[Ee][-+]?\d+`), floatValue())
-	lexer.Add([]byte(`-?\d+`), numberValue())
-
-	lexer.Add([]byte(`[Tt][Rr][Uu][Ee]`), booleanValue(true))
-	lexer.Add([]byte(`[Ff][Aa][Ll][Ss][Ee]`), booleanValue(false))
-
-	lexer.Add([]byte(`[Nn][Uu][Ll][Ll]`), nullValue())
-	lexer.Add([]byte(`~`), nullValue())
-
-	lexer.Add([]byte(`"[^"]*"`), stringValue(true))
-	lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true))
-	lexer.Add([]byte(`env\([^\)]+\)`), envOp(false))
-
-	lexer.Add([]byte(`\[`), literalToken(OpenCollect, false))
-	lexer.Add([]byte(`\]`), literalToken(CloseCollect, true))
-	lexer.Add([]byte(`\{`), literalToken(OpenCollectObject, false))
-	lexer.Add([]byte(`\}`), literalToken(CloseCollectObject, true))
-	lexer.Add([]byte(`\*`), opTokenWithPrefs(Multiply, nil, &MultiplyPreferences{AppendArrays: false}))
-	lexer.Add([]byte(`\*\+`), opTokenWithPrefs(Multiply, nil, &MultiplyPreferences{AppendArrays: true}))
-	lexer.Add([]byte(`\+`), opToken(Add))
-	lexer.Add([]byte(`\+=`), opToken(AddAssign))
-
-	err := lexer.Compile()
-	if err != nil {
-		return nil, err
-	}
-	return lexer, nil
-}
-
-type PathTokeniser interface {
-	Tokenise(path string) ([]*Token, error)
-}
-
-type pathTokeniser struct {
-	lexer *lex.Lexer
-}
-
-func NewPathTokeniser() PathTokeniser {
-	var lexer, err = initLexer()
-	if err != nil {
-		panic(err)
-	}
-	return &pathTokeniser{lexer}
-}
-
-func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) {
-	scanner, err := p.lexer.Scanner([]byte(path))
-
-	if err != nil {
-		return nil, fmt.Errorf("Parsing expression: %v", err)
-	}
-	var tokens []*Token
-	for tok, err, eof := scanner.Next(); !eof; tok, err, eof = scanner.Next() {
-
-		if tok != nil {
-			token := tok.(*Token)
-			log.Debugf("Tokenising %v", token.toString())
-			tokens = append(tokens, token)
-		}
-		if err != nil {
-			return nil, fmt.Errorf("Parsing expression: %v", err)
-		}
-	}
-	var postProcessedTokens = make([]*Token, 0)
-
-	skipNextToken := false
-
-	for index := range tokens {
-		if skipNextToken {
-			skipNextToken = false
-		} else {
-			postProcessedTokens, skipNextToken = p.handleToken(tokens, index, postProcessedTokens)
-		}
-	}
-
-	return postProcessedTokens, nil
-}
-
-func (p *pathTokeniser) handleToken(tokens []*Token, index int, postProcessedTokens []*Token) (tokensAccum []*Token, skipNextToken bool) {
-	skipNextToken = false
-	token := tokens[index]
-
-	if token.TokenType == TraverseArrayCollect {
-		//need to put a traverse array then a collect token
-		// do this by adding traverse then converting token to collect
-
-		op := &Operation{OperationType: TraverseArray, StringValue: "TRAVERSE_ARRAY"}
-		postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
-
-		token = &Token{TokenType: OpenCollect}
-
-	}
-
-	if index != len(tokens)-1 && token.AssignOperation != nil &&
-		tokens[index+1].TokenType == OperationToken &&
-		tokens[index+1].Operation.OperationType == Assign {
-		token.Operation = token.AssignOperation
-		token.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign
-		skipNextToken = true
-	}
-
-	postProcessedTokens = append(postProcessedTokens, token)
-
-	if index != len(tokens)-1 && token.CheckForPostTraverse &&
-		tokens[index+1].TokenType == OperationToken &&
-		tokens[index+1].Operation.OperationType == TraversePath {
-		op := &Operation{OperationType: ShortPipe, Value: "PIPE"}
-		postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
-	}
-	if index != len(tokens)-1 && token.CheckForPostTraverse &&
-		tokens[index+1].TokenType == OpenCollect {
-
-		op := &Operation{OperationType: ShortPipe, Value: "PIPE"}
-		postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
-
-		op = &Operation{OperationType: TraverseArray}
-		postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
-	}
-	if index != len(tokens)-1 && token.CheckForPostTraverse &&
-		tokens[index+1].TokenType == TraverseArrayCollect {
-
-		op := &Operation{OperationType: ShortPipe, Value: "PIPE"}
-		postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
-
-	}
-	return postProcessedTokens, skipNextToken
-}

pkg/yqlib/printer.go

@@ -24,6 +24,7 @@ type resultsPrinter struct {
 	previousDocIndex   uint
 	previousFileIndex  int
 	printedMatches     bool
+	treeNavigator      DataTreeNavigator
 }
 
 func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
@@ -35,6 +36,7 @@ func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEn
 		indent:             indent,
 		printDocSeparators: printDocSeparators,
 		firstTimePrinting:  true,
+		treeNavigator:      NewDataTreeNavigator(),
 	}
 }
 
@@ -74,9 +76,9 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	log.Debug("PrintResults for %v matches", matchingNodes.Len())
 	var err error
 	if p.outputToJSON {
-		explodeOp := Operation{OperationType: Explode}
-		explodeNode := PathTreeNode{Operation: &explodeOp}
-		matchingNodes, err = treeNavigator.GetMatchingNodes(matchingNodes, &explodeNode)
+		explodeOp := Operation{OperationType: explodeOpType}
+		explodeNode := ExpressionNode{Operation: &explodeOp}
+		matchingNodes, err = p.treeNavigator.GetMatchingNodes(matchingNodes, &explodeNode)
 		if err != nil {
 			return err
 		}

pkg/yqlib/stream_evaluator.go

@@ -8,24 +8,27 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
+// A yaml expression evaluator that runs the expression multiple times for each given yaml document.
+// Uses less memory than loading all documents and running the expression once, but this cannot process
+// cross document expressions.
 type StreamEvaluator interface {
-	Evaluate(filename string, reader io.Reader, node *PathTreeNode, printer Printer) error
+	Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer) error
 	EvaluateFiles(expression string, filenames []string, printer Printer) error
 	EvaluateNew(expression string, printer Printer) error
 }
 
 type streamEvaluator struct {
 	treeNavigator DataTreeNavigator
-	treeCreator   PathTreeCreator
+	treeCreator   ExpressionParser
 	fileIndex     int
 }
 
 func NewStreamEvaluator() StreamEvaluator {
-	return &streamEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewPathTreeCreator()}
+	return &streamEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewExpressionParser()}
 }
 
 func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error {
-	node, err := treeCreator.ParsePath(expression)
+	node, err := s.treeCreator.ParseExpression(expression)
 	if err != nil {
 		return err
 	}
@@ -38,7 +41,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error
 	inputList := list.New()
 	inputList.PushBack(candidateNode)
 
-	matches, errorParsing := treeNavigator.GetMatchingNodes(inputList, node)
+	matches, errorParsing := s.treeNavigator.GetMatchingNodes(inputList, node)
 	if errorParsing != nil {
 		return errorParsing
 	}
@@ -47,7 +50,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error
 
 func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
 
-	node, err := treeCreator.ParsePath(expression)
+	node, err := s.treeCreator.ParseExpression(expression)
 	if err != nil {
 		return err
 	}
@@ -70,7 +73,7 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p
 	return nil
 }
 
-func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *PathTreeNode, printer Printer) error {
+func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer) error {
 
 	var currentIndex uint
 
@@ -94,7 +97,7 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Path
 		inputList := list.New()
 		inputList.PushBack(candidateNode)
 
-		matches, errorParsing := treeNavigator.GetMatchingNodes(inputList, node)
+		matches, errorParsing := s.treeNavigator.GetMatchingNodes(inputList, node)
 		if errorParsing != nil {
 			return errorParsing
 		}

pkg/yqlib/utils.go

@@ -9,9 +9,6 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-var treeNavigator = NewDataTreeNavigator()
-var treeCreator = NewPathTreeCreator()
-
 func readStream(filename string) (io.Reader, error) {
 	if filename == "-" {
 		return bufio.NewReader(os.Stdin), nil

pkg/yqlib/write_in_place_handler.go

@@ -5,22 +5,22 @@ import (
 	"os"
 )
 
-type WriteInPlaceHandler interface {
+type writeInPlaceHandler interface {
 	CreateTempFile() (*os.File, error)
 	FinishWriteInPlace(evaluatedSuccessfully bool)
 }
 
-type writeInPlaceHandler struct {
+type writeInPlaceHandlerImpl struct {
 	inputFilename string
 	tempFile      *os.File
 }
 
-func NewWriteInPlaceHandler(inputFile string) WriteInPlaceHandler {
+func NewWriteInPlaceHandler(inputFile string) writeInPlaceHandler {
 
-	return &writeInPlaceHandler{inputFile, nil}
+	return &writeInPlaceHandlerImpl{inputFile, nil}
 }
 
-func (w *writeInPlaceHandler) CreateTempFile() (*os.File, error) {
+func (w *writeInPlaceHandlerImpl) CreateTempFile() (*os.File, error) {
 	info, err := os.Stat(w.inputFilename)
 	if err != nil {
 		return nil, err
@@ -46,7 +46,7 @@ func (w *writeInPlaceHandler) CreateTempFile() (*os.File, error) {
 	return file, err
 }
 
-func (w *writeInPlaceHandler) FinishWriteInPlace(evaluatedSuccessfully bool) {
+func (w *writeInPlaceHandlerImpl) FinishWriteInPlace(evaluatedSuccessfully bool) {
 	log.Debug("Going to write-inplace, evaluatedSuccessfully=%v, target=%v", evaluatedSuccessfully, w.inputFilename)
 	safelyCloseFile(w.tempFile)
 	if evaluatedSuccessfully {

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.3.1'
+version: '4.3.2'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.