Added gofmt to format command

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.9.0"
+	Version = "4.9.3"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.9.0
+FROM mikefarah/yq:4.9.3
 
 COPY entrypoint.sh /entrypoint.sh
 

pkg/yqlib/context.go

@@ -12,6 +12,14 @@ type Context struct {
 	DontAutoCreate bool
 }
 
+func (n *Context) SingleReadonlyChildContext(candidate *CandidateNode) Context {
+	list := list.New()
+	list.PushBack(candidate)
+	newContext := n.ChildContext(list)
+	newContext.DontAutoCreate = true
+	return newContext
+}
+
 func (n *Context) SingleChildContext(candidate *CandidateNode) Context {
 	list := list.New()
 	list.PushBack(candidate)
@@ -52,3 +60,9 @@ func (n *Context) Clone() Context {
 	}
 	return clone
 }
+
+func (n *Context) ReadOnlyClone() Context {
+	clone := n.Clone()
+	clone.DontAutoCreate = true
+	return clone
+}

pkg/yqlib/doc/Anchor and Alias Operators.md

@@ -169,6 +169,22 @@ b: &meow purr
 a: *meow
 ```
 
+## Set alias to blank does nothing
+Given a sample.yml file of:
+```yaml
+b: &meow purr
+a: cat
+```
+then
+```bash
+yq eval '.a alias = ""' sample.yml
+```
+will output
+```yaml
+b: &meow purr
+a: cat
+```
+
 ## Set alias relatively using assign-update
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/aa.md

@@ -71,7 +71,7 @@ dog
 Both sides have now been evaluated, so now the operator copies across the value from the RHS to the value on the LHS, and it returns the now updated context:
 
 ```yaml
-a: cat
+a: dog
 b: dog
 ```
 

pkg/yqlib/expression_postfix.go

@@ -54,7 +54,7 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			opStack = opStack[0 : len(opStack)-1]
 			log.Debugf("deleteing open bracket from opstack")
 
-			//and append a collect to the opStack
+			//and append a collect to the result
 			// hack - see if there's the optional traverse flag
 			// on the close op - move it to the collect op.
 			// allows for .["cat"]?
@@ -68,6 +68,12 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			result = append(result, &Operation{OperationType: shortPipeOpType})
 			log.Debugf("put shortpipe onto the result")
 
+			//traverseArrayCollect is a sneaky op that needs to be included too
+			//when closing a []
+			if len(opStack) > 0 && opStack[len(opStack)-1].Operation != nil && opStack[len(opStack)-1].Operation.OperationType == traverseArrayOpType {
+				opStack, result = popOpToResult(opStack, result)
+			}
+
 		case closeBracket:
 			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != openBracket {
 				opStack, result = popOpToResult(opStack, result)

pkg/yqlib/expression_processing_test.go

@@ -12,6 +12,21 @@ var pathTests = []struct {
 	expectedTokens  []interface{}
 	expectedPostFix []interface{}
 }{
+	{
+		`.[0]`,
+		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
+		append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
+	{
+		`.[0][1]`,
+		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]", "TRAVERSE_ARRAY", "[", "1 (int64)", "]"),
+		append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "1 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
+	{
+		`"\""`,
+		append(make([]interface{}, 0), "\" (string)"),
+		append(make([]interface{}, 0), "\" (string)"),
+	},
 	{
 		`[]|join(".")`,
 		append(make([]interface{}, 0), "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")"),

pkg/yqlib/expression_tokeniser.go

@@ -189,6 +189,7 @@ func stringValue(wrapped bool) lex.Action {
 		if wrapped {
 			value = unwrap(value)
 		}
+		value = strings.ReplaceAll(value, "\\\"", "\"")
 		return &token{TokenType: operationToken, Operation: createValueOperation(value, value)}, nil
 	}
 }
@@ -334,7 +335,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`[Nn][Uu][Ll][Ll]`), nullValue())
 	lexer.Add([]byte(`~`), nullValue())
 
-	lexer.Add([]byte(`"[^"]*"`), stringValue(true))
+	lexer.Add([]byte(`"([^"\\]*(\\.[^"\\]*)*)"`), stringValue(true))
 	lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true))
 	lexer.Add([]byte(`env\([^\)]+\)`), envOp(false))
 
@@ -410,16 +411,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	skipNextToken = false
 	currentToken := tokens[index]
 
+	log.Debug("processing %v", currentToken.toString(true))
+
 	if currentToken.TokenType == traverseArrayCollect {
 		//need to put a traverse array then a collect currentToken
 		// do this by adding traverse then converting currentToken to collect
 
 		if index == 0 || tokens[index-1].TokenType != operationToken ||
 			tokens[index-1].Operation.OperationType != traversePathOpType {
+			log.Debug("  adding self")
 			op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"}
 			postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 		}
-
+		log.Debug("  adding traverse array")
 		op := &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 
@@ -430,17 +434,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	if index != len(tokens)-1 && currentToken.AssignOperation != nil &&
 		tokens[index+1].TokenType == operationToken &&
 		tokens[index+1].Operation.OperationType == assignOpType {
+		log.Debug("  its an update assign")
 		currentToken.Operation = currentToken.AssignOperation
 		currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign
 		skipNextToken = true
 	}
 
+	log.Debug("  adding token to the fixed list")
 	postProcessedTokens = append(postProcessedTokens, currentToken)
 
 	if index != len(tokens)-1 &&
 		((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) ||
 			(currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) {
-
+		log.Debug("  adding empty")
 		op := &Operation{OperationType: emptyOpType, StringValue: "EMPTY"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}
@@ -448,12 +454,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
 		tokens[index+1].TokenType == operationToken &&
 		tokens[index+1].Operation.OperationType == traversePathOpType {
+		log.Debug("  adding pipe because the next thing is traverse")
 		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}
 	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
 		tokens[index+1].TokenType == openCollect {
 
+		// if tokens[index].TokenType == closeCollect {
+		// 	log.Debug("  adding pipe because next is opencollect")
+		// 	op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
+		// 	postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+		// }
+		log.Debug("  adding traverArray because next is opencollect")
 		op := &Operation{OperationType: traverseArrayOpType}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}

pkg/yqlib/lib.go

@@ -20,9 +20,6 @@ type operationType struct {
 	Handler    operatorHandler
 }
 
-// operators TODO:
-// - mergeEmpty (sets only if the document is empty, do I do that now?)
-
 var orOpType = &operationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: orOperator}
 var andOpType = &operationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: andOperator}
 var reduceOpType = &operationType{Type: "REDUCE", NumArgs: 2, Precedence: 35, Handler: reduceOperator}

pkg/yqlib/operator_add.go

@@ -39,7 +39,7 @@ func toNodes(candidate *CandidateNode) []*yaml.Node {
 func addOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("Add operator")
 
-	return crossFunction(d, context, expressionNode, add, false)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, add, false)
 }
 
 func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

pkg/yqlib/operator_add_test.go

@@ -14,6 +14,22 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[1 a], (!!int)::3\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: "(.a + .b) as $x",
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: 0`,
+		expression: ".a += .b.c",
+		expected: []string{
+			"D0, P[], (doc)::a: 0\n",
+		},
+	},
 	{
 		description: "Concatenate and assign arrays",
 		document:    `{a: {val: thing, b: [cat,dog]}}`,

pkg/yqlib/operator_alternative.go

@@ -2,7 +2,7 @@ package yqlib
 
 func alternativeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- alternative")
-	return crossFunction(d, context, expressionNode, alternativeFunc, true)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, alternativeFunc, true)
 }
 
 func alternativeFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

pkg/yqlib/operator_alternative_test.go

@@ -5,6 +5,20 @@ import (
 )
 
 var alternativeOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		expression: `.b // .c`,
+		document:   `a: bridge`,
+		expected:   []string{},
+	},
+	{
+		skipDoc:    true,
+		expression: `(.b // "hello") as $x`,
+		document:   `a: bridge`,
+		expected: []string{
+			"D0, P[], (doc)::a: bridge\n",
+		},
+	},
 	{
 		description: "LHS is defined",
 		expression:  `.a // "hello"`,

pkg/yqlib/operator_anchors_aliases.go

@@ -12,7 +12,7 @@ func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *
 
 	aliasName := ""
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -32,7 +32,7 @@ func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *
 		log.Debugf("Setting aliasName : %v", candidate.GetKey())
 
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}
@@ -41,9 +41,11 @@ func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *
 			}
 		}
 
+		if aliasName != "" {
 			candidate.Node.Kind = yaml.AliasNode
 			candidate.Node.Value = aliasName
 		}
+	}
 	return context, nil
 }
 
@@ -66,7 +68,7 @@ func assignAnchorOperator(d *dataTreeNavigator, context Context, expressionNode
 
 	anchorName := ""
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -87,7 +89,7 @@ func assignAnchorOperator(d *dataTreeNavigator, context Context, expressionNode
 		log.Debugf("Setting anchorName of : %v", candidate.GetKey())
 
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}

pkg/yqlib/operator_anchors_aliases_test.go

@@ -58,6 +58,22 @@ var anchorOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: &cat {b: cat}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `a: {c: cat}`,
+		expression: `.a anchor |= .b`,
+		expected: []string{
+			"D0, P[], (doc)::a: {c: cat}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: {c: cat}`,
+		expression: `.a anchor = .b`,
+		expected: []string{
+			"D0, P[], (doc)::a: {c: cat}\n",
+		},
+	},
 	{
 		description: "Get alias",
 		document:    `{b: &billyBob meow, a: *billyBob}`,
@@ -74,6 +90,30 @@ var anchorOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{b: &meow purr, a: *meow}\n",
 		},
 	},
+	{
+		description: "Set alias to blank does nothing",
+		document:    `{b: &meow purr, a: cat}`,
+		expression:  `.a alias = ""`,
+		expected: []string{
+			"D0, P[], (doc)::{b: &meow purr, a: cat}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{b: &meow purr, a: cat}`,
+		expression: `.a alias = .c`,
+		expected: []string{
+			"D0, P[], (doc)::{b: &meow purr, a: cat}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{b: &meow purr, a: cat}`,
+		expression: `.a alias |= .c`,
+		expected: []string{
+			"D0, P[], (doc)::{b: &meow purr, a: cat}\n",
+		},
+	},
 	{
 		description: "Set alias relatively using assign-update",
 		document:    `{b: &meow purr, a: {f: meow}}`,
@@ -159,7 +199,7 @@ foobar:
 	},
 }
 
-func TestAnchorAliaseOperatorScenarios(t *testing.T) {
+func TestAnchorAliasOperatorScenarios(t *testing.T) {
 	for _, tt := range anchorOperatorScenarios {
 		testScenario(t, &tt)
 	}

pkg/yqlib/operator_assign.go

@@ -7,7 +7,7 @@ func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode
 	}
 	var rhs Context
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err = d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err = d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 	}
 
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
@@ -44,7 +44,7 @@ func assignAttributesOperator(d *dataTreeNavigator, context Context, expressionN
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 
 		if err != nil {
 			return Context{}, err

pkg/yqlib/operator_assign_test.go

@@ -12,6 +12,22 @@ var assignOperatorScenarios = []expressionScenario{
 			"D0, P[], ()::a:\n    b: cat\nx: frog\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: `.a |= .b`,
+		expected: []string{
+			"D0, P[], (doc)::{a: null}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: `.a = .b`,
+		expected: []string{
+			"D0, P[], (doc)::{a: null}\n",
+		},
+	},
 	{
 		description: "Update node to be the child value",
 		document:    `{a: {b: {g: foof}}}`,

pkg/yqlib/operator_booleans.go

@@ -71,7 +71,7 @@ func findBoolean(wantBool bool, d *dataTreeNavigator, context Context, expressio
 		if expressionNode != nil {
 			//need to evaluate the expression against the node
 			candidate := &CandidateNode{Node: node}
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode)
 			if err != nil {
 				return false, err
 			}
@@ -134,7 +134,7 @@ func anyOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
 
 func orOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- orOp")
-	return crossFunction(d, context, expressionNode, performBoolOp(
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, performBoolOp(
 		func(b1 bool, b2 bool) bool {
 			log.Debugf("-- peformingOrOp with %v and %v", b1, b2)
 			return b1 || b2
@@ -143,7 +143,7 @@ func orOperator(d *dataTreeNavigator, context Context, expressionNode *Expressio
 
 func andOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- AndOp")
-	return crossFunction(d, context, expressionNode, performBoolOp(
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, performBoolOp(
 		func(b1 bool, b2 bool) bool {
 			return b1 && b2
 		}), true)

pkg/yqlib/operator_booleans_test.go

@@ -12,6 +12,14 @@ var booleanOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::true\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "b: hi",
+		expression: `.a or .c`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "b: hi",
@@ -67,6 +75,22 @@ var booleanOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: true\nb: false\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `[{pet: cat}]`,
+		expression: `any_c(.name == "harry") as $c`,
+		expected: []string{
+			"D0, P[], (doc)::[{pet: cat}]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{pet: cat}]`,
+		expression: `all_c(.name == "harry") as $c`,
+		expected: []string{
+			"D0, P[], (doc)::[{pet: cat}]\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `[false, false]`,
@@ -125,6 +149,22 @@ var booleanOperatorScenarios = []expressionScenario{
 			"D0, P[b], (!!bool)::true\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: `(.a.b or .c) as $x`,
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: `(.a.b and .c) as $x`,
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
 	{
 		description: "Not true is false",
 		expression:  `true | not`,

pkg/yqlib/operator_collect_object.go

@@ -17,9 +17,12 @@ import (
 ...
 */
 
-func collectObjectOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+func collectObjectOperator(d *dataTreeNavigator, originalContext Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- collectObjectOperation")
 
+	context := originalContext.Clone()
+	context.DontAutoCreate = false
+
 	if context.MatchingNodes.Len() == 0 {
 		node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map", Value: "{}"}
 		candidate := &CandidateNode{Node: node}

pkg/yqlib/operator_collect_object_test.go

@@ -5,6 +5,30 @@ import (
 )
 
 var collectObjectOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   "a: []",
+		expression: `.a += [{"key": "att2", "value": "val2"}]`,
+		expected: []string{
+			"D0, P[], (doc)::a: [{key: att2, value: val2}]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "",
+		expression: `.a += {"key": "att2", "value": "val2"}`,
+		expected: []string{
+			"D0, P[], ()::a:\n    key: att2\n    value: val2\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "",
+		expression: `.a += [0]`,
+		expected: []string{
+			"D0, P[], ()::a:\n    - 0\n",
+		},
+	},
 	{
 		description: `Collect empty object`,
 		document:    ``,

pkg/yqlib/operator_collect_test.go

@@ -5,6 +5,14 @@ import (
 )
 
 var collectOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   ``,
+		expression: `.a += [0]`,
+		expected: []string{
+			"D0, P[], ()::a:\n    - 0\n",
+		},
+	},
 	{
 		description: "Collect empty",
 		document:    ``,

pkg/yqlib/operator_comments.go

@@ -27,7 +27,7 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
 
 	comment := ""
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -41,7 +41,7 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
 		candidate := el.Value.(*CandidateNode)
 
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}

pkg/yqlib/operator_comments_test.go

@@ -62,6 +62,22 @@ var commentOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: cat\n\n# cat\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `a: cat`,
+		expression: `. footComment=.b.d`,
+		expected: []string{
+			"D0, P[], (doc)::a: cat\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: cat`,
+		expression: `. footComment|=.b.d`,
+		expected: []string{
+			"D0, P[], (doc)::a: cat\n",
+		},
+	},
 	{
 		description: "Remove comment",
 		document:    "a: cat # comment\nb: dog # leave this",

pkg/yqlib/operator_delete.go

@@ -7,9 +7,7 @@ import (
 )
 
 func deleteChildOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	contextToUse := context.Clone()
-	contextToUse.DontAutoCreate = true
-	nodesToDelete, err := d.GetMatchingNodes(contextToUse, expressionNode.Rhs)
+	nodesToDelete, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 
 	if err != nil {
 		return Context{}, err

pkg/yqlib/operator_equals.go

@@ -4,20 +4,32 @@ import "gopkg.in/yaml.v3"
 
 func equalsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- equalsOperation")
-	return crossFunction(d, context, expressionNode, isEquals(false), true)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, isEquals(false), true)
 }
 
 func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		value := false
-
+		log.Debugf("-- isEquals cross function")
 		if lhs == nil && rhs == nil {
 			owner := &CandidateNode{}
 			return createBooleanCandidate(owner, !flip), nil
 		} else if lhs == nil {
-			return createBooleanCandidate(rhs, flip), nil
+			log.Debugf("lhs nil, but rhs is not")
+			rhsNode := unwrapDoc(rhs.Node)
+			value := rhsNode.Tag == "!!null"
+			if flip {
+				value = !value
+			}
+			return createBooleanCandidate(rhs, value), nil
 		} else if rhs == nil {
-			return createBooleanCandidate(lhs, flip), nil
+			log.Debugf("lhs not nil, but rhs is")
+			lhsNode := unwrapDoc(lhs.Node)
+			value := lhsNode.Tag == "!!null"
+			if flip {
+				value = !value
+			}
+			return createBooleanCandidate(lhs, value), nil
 		}
 
 		lhsNode := unwrapDoc(lhs.Node)
@@ -37,6 +49,6 @@ func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *Candid
 }
 
 func notEqualsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	log.Debugf("-- equalsOperation")
-	return crossFunction(d, context, expressionNode, isEquals(true), true)
+	log.Debugf("-- notEqualsOperator")
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, isEquals(true), true)
 }

pkg/yqlib/operator_equals_test.go

@@ -5,6 +5,13 @@ import (
 )
 
 var equalsOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		expression: ".a == .b",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "cat",
@@ -14,6 +21,60 @@ var equalsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::false\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: "(.a == .b) as $x",
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: ".a == .b",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: "(.a != .b) as $x",
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: ".a != .b",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{a: {b: 10}}",
+		expression: "select(.c != null)",
+		expected:   []string{},
+	},
+	{
+		skipDoc:    true,
+		document:   "{a: {b: 10}}",
+		expression: "select(.d == .c)",
+		expected: []string{
+			"D0, P[], (doc)::{a: {b: 10}}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{a: {b: 10}}",
+		expression: "select(null == .c)",
+		expected: []string{
+			"D0, P[], (doc)::{a: {b: 10}}\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "{a: { b: {things: \"\"}, f: [1], g: [] }}",

pkg/yqlib/operator_has.go

@@ -12,14 +12,19 @@ func hasOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	log.Debugf("-- hasOperation")
 	var results = list.New()
 
-	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
-	wanted := rhs.MatchingNodes.Front().Value.(*CandidateNode).Node
-	wantedKey := wanted.Value
+	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 
 	if err != nil {
 		return Context{}, err
 	}
 
+	wantedKey := "null"
+	wanted := &yaml.Node{Tag: "!!null"}
+	if rhs.MatchingNodes.Len() != 0 {
+		wanted = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node
+		wantedKey = wanted.Value
+	}
+
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 

pkg/yqlib/operator_has_test.go

@@ -13,6 +13,22 @@ var hasOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::true\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `a: hello`,
+		expression: `has(.b) as $c`,
+		expected: []string{
+			"D0, P[], (doc)::a: hello\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: hello`,
+		expression: `has(.b)`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
 	{
 		description: "Has map key",
 		document: `- a: "yes"

pkg/yqlib/operator_select.go

@@ -11,9 +11,7 @@ func selectOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		childContext := context.SingleChildContext(candidate)
-		childContext.DontAutoCreate = true
-		rhs, err := d.GetMatchingNodes(childContext, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 
 		if err != nil {
 			return Context{}, err
@@ -24,7 +22,9 @@ func selectOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 
 		if first != nil {
 			result := first.Value.(*CandidateNode)
+			log.Debugf("result %v", NodeToString(result))
 			includeResult, errDecoding := isTruthy(result)
+			log.Debugf("isTruthy %v", includeResult)
 			if errDecoding != nil {
 				return Context{}, errDecoding
 			}

pkg/yqlib/operator_sort_keys.go

@@ -10,7 +10,7 @@ func sortKeysOperator(d *dataTreeNavigator, context Context, expressionNode *Exp
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}

pkg/yqlib/operator_sort_keys_test.go

@@ -13,6 +13,14 @@ var sortKeysOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: blah, b: bing, c: frog}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{c: frog}`,
+		expression: `sortKeys(.d)`,
+		expected: []string{
+			"D0, P[], (doc)::{c: frog}\n",
+		},
+	},
 	{
 		description:    "Sort keys recursively",
 		subdescription: "Note the array elements are left unsorted, but maps inside arrays are sorted",

pkg/yqlib/operator_strings.go

@@ -13,7 +13,7 @@ func getSubstituteParameters(d *dataTreeNavigator, block *ExpressionNode, contex
 	regEx := ""
 	replacementText := ""
 
-	regExNodes, err := d.GetMatchingNodes(context, block.Lhs)
+	regExNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), block.Lhs)
 	if err != nil {
 		return "", "", err
 	}
@@ -78,7 +78,7 @@ func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *E
 	log.Debugf("-- joinStringOperator")
 	joinStr := ""
 
-	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 	if err != nil {
 		return Context{}, err
 	}
@@ -119,7 +119,7 @@ func splitStringOperator(d *dataTreeNavigator, context Context, expressionNode *
 	log.Debugf("-- splitStringOperator")
 	splitStr := ""
 
-	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 	if err != nil {
 		return Context{}, err
 	}

pkg/yqlib/operator_style.go

@@ -31,7 +31,7 @@ func assignStyleOperator(d *dataTreeNavigator, context Context, expressionNode *
 	log.Debugf("AssignStyleOperator: %v")
 	var style yaml.Style
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -54,7 +54,7 @@ func assignStyleOperator(d *dataTreeNavigator, context Context, expressionNode *
 		candidate := el.Value.(*CandidateNode)
 		log.Debugf("Setting style of : %v", candidate.GetKey())
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}

pkg/yqlib/operator_subtract.go

@@ -25,7 +25,7 @@ func subtractAssignOperator(d *dataTreeNavigator, context Context, expressionNod
 func subtractOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("Subtract operator")
 
-	return crossFunction(d, context, expressionNode, subtract, false)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, subtract, false)
 }
 
 func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

pkg/yqlib/operator_subtract_test.go

@@ -5,6 +5,14 @@ import (
 )
 
 var subtractOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: "(.a - .b) as $x",
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
 	{
 		description:    "Number subtraction - float",
 		subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.",

pkg/yqlib/operator_tag.go

@@ -12,7 +12,7 @@ func assignTagOperator(d *dataTreeNavigator, context Context, expressionNode *Ex
 	tag := ""
 
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -32,7 +32,7 @@ func assignTagOperator(d *dataTreeNavigator, context Context, expressionNode *Ex
 		candidate := el.Value.(*CandidateNode)
 		log.Debugf("Setting tag of : %v", candidate.GetKey())
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}

pkg/yqlib/operator_traverse_path.go

@@ -89,7 +89,7 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
 
 	// rhs is a collect expression that will yield indexes to retreive of the arrays
 
-	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 
 	if err != nil {
 		return Context{}, err
@@ -308,6 +308,6 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *C
 
 func traverseArray(candidate *CandidateNode, operation *Operation, prefs traversePreferences) (*list.List, error) {
 	log.Debug("operation Value %v", operation.Value)
-	indices := []*yaml.Node{&yaml.Node{Value: operation.StringValue}}
+	indices := []*yaml.Node{{Value: operation.StringValue}}
 	return traverseArrayWithIndices(candidate, indices, prefs)
 }

pkg/yqlib/operator_traverse_path_test.go

@@ -27,6 +27,22 @@ foobar:
 `
 
 var traversePathOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `[[1]]`,
+		expression: `.[0][0]`,
+		expected: []string{
+			"D0, P[0 0], (!!int)::1\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[[[1]]]`,
+		expression: `.[0][0][0]`,
+		expected: []string{
+			"D0, P[0 0 0], (!!int)::1\n",
+		},
+	},
 	{
 		description: "Simple map navigation",
 		document:    `{a: {b: apple}}`,
@@ -78,6 +94,14 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[flying fox], (!!str)::frog\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `c: dog`,
+		expression: `.[.a.b] as $x`,
+		expected: []string{
+			"D0, P[], (doc)::c: dog\n",
+		},
+	},
 	{
 		description:    "Dynamic keys",
 		subdescription: `Expressions within [] can be used to dynamically lookup / calculate keys`,

pkg/yqlib/operator_union_test.go

@@ -5,6 +5,20 @@ import (
 )
 
 var unionOperatorScenarios = []expressionScenario{
+	// {
+	// 	skipDoc:    true,
+	// 	document:   "{}",
+	// 	expression: `(.a, .b.c) as $x`,
+	// 	expected: []string{
+	// 		"D0, P[], (doc)::{}\n",
+	// 	},
+	// },
+	// {
+	// 	skipDoc:    true,
+	// 	document:   "{}",
+	// 	expression: `(.a, .b.c)`,
+	// 	expected:   []string{},
+	// },
 	{
 		description: "Combine scalars",
 		expression:  `1, true, "cat"`,

pkg/yqlib/operator_unique.go

@@ -31,15 +31,20 @@ func uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionN
 		var newMatches = orderedmap.NewOrderedMap()
 		for _, node := range candidateNode.Content {
 			child := &CandidateNode{Node: node}
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(child), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(child), expressionNode.Rhs)
 
 			if err != nil {
 				return Context{}, err
 			}
 
+			keyValue := "null"
+
+			if rhs.MatchingNodes.Len() > 0 {
 				first := rhs.MatchingNodes.Front()
 				keyCandidate := first.Value.(*CandidateNode)
-			keyValue := keyCandidate.Node.Value
+				keyValue = keyCandidate.Node.Value
+			}
+
 			_, exists := newMatches.Get(keyValue)
 
 			if !exists {

pkg/yqlib/operator_unique_test.go

@@ -39,6 +39,22 @@ var uniqueOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::- {name: harry, pet: cat}\n- {name: billy, pet: dog}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`,
+		expression: `unique_by(.name)`,
+		expected: []string{
+			"D0, P[], (!!seq)::- {name: harry, pet: cat}\n- {pet: fish}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`,
+		expression: `unique_by(.cat.dog)`,
+		expected: []string{
+			"D0, P[], (!!seq)::- {name: harry, pet: cat}\n",
+		},
+	},
 }
 
 func TestUniqueOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_variables.go

@@ -16,7 +16,7 @@ func getVariableOperator(d *dataTreeNavigator, context Context, expressionNode *
 }
 
 func assignVariableOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
+	lhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Lhs)
 	if err != nil {
 		return Context{}, nil
 	}

pkg/yqlib/operator_variables_test.go

@@ -5,6 +5,14 @@ import (
 )
 
 var variableOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: `.a.b as $foo`,
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
 	{
 		description: "Single value variable",
 		document:    `a: cat`,

pkg/yqlib/operators.go

@@ -24,7 +24,19 @@ func emptyOperator(d *dataTreeNavigator, context Context, expressionNode *Expres
 
 type crossFunctionCalculation func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)
 
-func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *CandidateNode, rhs Context, calculation crossFunctionCalculation, results *list.List) error {
+func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *CandidateNode, rhs Context, calculation crossFunctionCalculation, results *list.List, calcWhenEmpty bool) error {
+
+	if calcWhenEmpty && rhs.MatchingNodes.Len() == 0 {
+		resultCandidate, err := calculation(d, context, lhsCandidate, nil)
+		if err != nil {
+			return err
+		}
+		if resultCandidate != nil {
+			results.PushBack(resultCandidate)
+		}
+		return nil
+	}
+
 	for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() {
 		log.Debugf("Applying calc")
 		rhsCandidate := rightEl.Value.(*CandidateNode)
@@ -32,8 +44,10 @@ func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *Candidat
 		if err != nil {
 			return err
 		}
+		if resultCandidate != nil {
 			results.PushBack(resultCandidate)
 		}
+	}
 	return nil
 }
 
@@ -52,14 +66,7 @@ func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	}
 
 	if calcWhenEmpty && lhs.MatchingNodes.Len() == 0 {
-		if rhs.MatchingNodes.Len() == 0 {
-			resultCandidate, err := calculation(d, context, nil, nil)
-			if err != nil {
-				return Context{}, err
-			}
-			results.PushBack(resultCandidate)
-		}
-		err := resultsForRhs(d, context, nil, rhs, calculation, results)
+		err := resultsForRhs(d, context, nil, rhs, calculation, results, calcWhenEmpty)
 		if err != nil {
 			return Context{}, err
 		}
@@ -68,7 +75,7 @@ func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
 		lhsCandidate := el.Value.(*CandidateNode)
 
-		err := resultsForRhs(d, context, lhsCandidate, rhs, calculation, results)
+		err := resultsForRhs(d, context, lhsCandidate, rhs, calculation, results, calcWhenEmpty)
 		if err != nil {
 			return Context{}, err
 		}

scripts/format.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
 find . \( -path ./vendor \) -prune -o -name "*.go" -exec goimports -w {} \;
+gofmt -w -s .
 go mod tidy
 go mod vendor

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.9.0'
+version: '4.9.3'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.