Added gofmt to format command

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.7.1"
+	Version = "4.9.3"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.7.1
+FROM mikefarah/yq:4.9.3
 
 COPY entrypoint.sh /entrypoint.sh
 

pkg/yqlib/context.go

@@ -12,6 +12,14 @@ type Context struct {
 	DontAutoCreate bool
 }
 
+func (n *Context) SingleReadonlyChildContext(candidate *CandidateNode) Context {
+	list := list.New()
+	list.PushBack(candidate)
+	newContext := n.ChildContext(list)
+	newContext.DontAutoCreate = true
+	return newContext
+}
+
 func (n *Context) SingleChildContext(candidate *CandidateNode) Context {
 	list := list.New()
 	list.PushBack(candidate)
@@ -52,3 +60,9 @@ func (n *Context) Clone() Context {
 	}
 	return clone
 }
+
+func (n *Context) ReadOnlyClone() Context {
+	clone := n.Clone()
+	clone.DontAutoCreate = true
+	return clone
+}

pkg/yqlib/doc/Anchor and Alias Operators.md

@@ -3,6 +3,97 @@ Use the `alias` and `anchor` operators to read and write yaml aliases and anchor
 `yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
 
 
+## Merge one map
+see https://yaml.org/type/merge.html
+
+Given a sample.yml file of:
+```yaml
+- &CENTER
+  x: 1
+  y: 2
+- &LEFT
+  x: 0
+  y: 2
+- &BIG
+  r: 10
+- &SMALL
+  r: 1
+- !!merge <<: *CENTER
+  r: 10
+```
+then
+```bash
+yq eval '.[4] | explode(.)' sample.yml
+```
+will output
+```yaml
+x: 1
+y: 2
+r: 10
+```
+
+## Merge multiple maps
+see https://yaml.org/type/merge.html
+
+Given a sample.yml file of:
+```yaml
+- &CENTER
+  x: 1
+  y: 2
+- &LEFT
+  x: 0
+  y: 2
+- &BIG
+  r: 10
+- &SMALL
+  r: 1
+- !!merge <<:
+    - *CENTER
+    - *BIG
+```
+then
+```bash
+yq eval '.[4] | explode(.)' sample.yml
+```
+will output
+```yaml
+r: 10
+x: 1
+y: 2
+```
+
+## Override
+see https://yaml.org/type/merge.html
+
+Given a sample.yml file of:
+```yaml
+- &CENTER
+  x: 1
+  y: 2
+- &LEFT
+  x: 0
+  y: 2
+- &BIG
+  r: 10
+- &SMALL
+  r: 1
+- !!merge <<:
+    - *BIG
+    - *LEFT
+    - *SMALL
+  x: 1
+```
+then
+```bash
+yq eval '.[4] | explode(.)' sample.yml
+```
+will output
+```yaml
+r: 10
+x: 1
+y: 2
+```
+
 ## Get anchor
 Given a sample.yml file of:
 ```yaml
@@ -78,6 +169,22 @@ b: &meow purr
 a: *meow
 ```
 
+## Set alias to blank does nothing
+Given a sample.yml file of:
+```yaml
+b: &meow purr
+a: cat
+```
+then
+```bash
+yq eval '.a alias = ""' sample.yml
+```
+will output
+```yaml
+b: &meow purr
+a: cat
+```
+
 ## Set alias relatively using assign-update
 Given a sample.yml file of:
 ```yaml
@@ -183,9 +290,9 @@ bar:
   c: bar_c
 foobarList:
   b: bar_b
-  a: foo_a
+  thing: foo_thing
-  thing: bar_thing
   c: foobarList_c
+  a: foo_a
 foobar:
   c: foo_c
   a: foo_a

pkg/yqlib/doc/Boolean Operators.md

@@ -1,5 +1,14 @@
-The `or` and `and` operators take two parameters and return a boolean result. `not` flips a boolean from true to false, or vice versa. These are most commonly used with the `select` operator to filter particular nodes.
+The `or` and `and` operators take two parameters and return a boolean result. 
-## OR example
+
+`not` flips a boolean from true to false, or vice versa. 
+
+`any` will return `true` if there are any `true` values in a array sequence, and `all` will return true if _all_ elements in an array are true.
+
+`any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet
+
+These are most commonly used with the `select` operator to filter particular nodes.
+
+## `or` example
 Running
 ```bash
 yq eval --null-input 'true or false'
@@ -9,7 +18,7 @@ will output
 true
 ```
 
-## AND example
+## `and` example
 Running
 ```bash
 yq eval --null-input 'true and false'
@@ -41,6 +50,104 @@ will output
   b: fly
 ```
 
+## `any` returns true if any boolean in a given array is true
+Given a sample.yml file of:
+```yaml
+- false
+- true
+```
+then
+```bash
+yq eval 'any' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## `any` returns false for an empty array
+Given a sample.yml file of:
+```yaml
+[]
+```
+then
+```bash
+yq eval 'any' sample.yml
+```
+will output
+```yaml
+false
+```
+
+## `any_c` returns true if any element in the array is true for the given condition.
+Given a sample.yml file of:
+```yaml
+a:
+  - rad
+  - awesome
+b:
+  - meh
+  - whatever
+```
+then
+```bash
+yq eval '.[] |= any_c(. == "awesome")' sample.yml
+```
+will output
+```yaml
+a: true
+b: false
+```
+
+## `all` returns true if all booleans in a given array are true
+Given a sample.yml file of:
+```yaml
+- true
+- true
+```
+then
+```bash
+yq eval 'all' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## `all` returns true for an empty array
+Given a sample.yml file of:
+```yaml
+[]
+```
+then
+```bash
+yq eval 'all' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## `all_c` returns true if all elements in the array are true for the given condition.
+Given a sample.yml file of:
+```yaml
+a:
+  - rad
+  - awesome
+b:
+  - meh
+  - 12
+```
+then
+```bash
+yq eval '.[] |= all_c(tag == "!!str")' sample.yml
+```
+will output
+```yaml
+a: true
+b: false
+```
+
 ## Not true is false
 Running
 ```bash

pkg/yqlib/doc/Comment Operators.md

@@ -88,7 +88,7 @@ a: cat
 b: dog # leave this
 ```
 
-## Remove all comments
+## Remove (strip) all comments
 Note the use of `...` to ensure key nodes are included.
 
 Given a sample.yml file of:

pkg/yqlib/doc/Entries.md

@@ -0,0 +1,100 @@
+Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps.
+## to_entries Map
+Given a sample.yml file of:
+```yaml
+a: 1
+b: 2
+```
+then
+```bash
+yq eval 'to_entries' sample.yml
+```
+will output
+```yaml
+- key: a
+  value: 1
+- key: b
+  value: 2
+```
+
+## to_entries Array
+Given a sample.yml file of:
+```yaml
+- a
+- b
+```
+then
+```bash
+yq eval 'to_entries' sample.yml
+```
+will output
+```yaml
+- key: 0
+  value: a
+- key: 1
+  value: b
+```
+
+## to_entries null
+Given a sample.yml file of:
+```yaml
+null
+```
+then
+```bash
+yq eval 'to_entries' sample.yml
+```
+will output
+```yaml
+```
+
+## from_entries map
+Given a sample.yml file of:
+```yaml
+a: 1
+b: 2
+```
+then
+```bash
+yq eval 'to_entries | from_entries' sample.yml
+```
+will output
+```yaml
+a: 1
+b: 2
+```
+
+## from_entries with numeric key indexes
+from_entries always creates a map, even for numeric keys
+
+Given a sample.yml file of:
+```yaml
+- a
+- b
+```
+then
+```bash
+yq eval 'to_entries | from_entries' sample.yml
+```
+will output
+```yaml
+0: a
+1: b
+```
+
+## Use with_entries to update keys
+Given a sample.yml file of:
+```yaml
+a: 1
+b: 2
+```
+then
+```bash
+yq eval 'with_entries(.key |= "KEY_" + .)' sample.yml
+```
+will output
+```yaml
+KEY_a: 1
+KEY_b: 2
+```
+

pkg/yqlib/doc/Equals.md

@@ -93,3 +93,31 @@ will output
 true
 ```
 
+## Non exisitant key doesn't equal a value
+Given a sample.yml file of:
+```yaml
+a: frog
+```
+then
+```bash
+yq eval 'select(.b != "thing")' sample.yml
+```
+will output
+```yaml
+a: frog
+```
+
+## Two non existant keys are equal
+Given a sample.yml file of:
+```yaml
+a: frog
+```
+then
+```bash
+yq eval 'select(.b == .c)' sample.yml
+```
+will output
+```yaml
+a: frog
+```
+

pkg/yqlib/doc/Multiply (Merge).md

@@ -231,6 +231,46 @@ will output
   age: 32
 ```
 
+## Merge arrays of objects together, matching on a key
+It's a complex command, the trickyness comes from needing to have the right context in the expressions.
+First we save the second array into a variable '$two' which lets us reference it later.
+We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. 
+We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter.
+
+
+Given a sample.yml file of:
+```yaml
+- a: apple
+  b: appleB
+- a: kiwi
+  b: kiwiB
+- a: banana
+  b: bananaB
+```
+And another sample another.yml file of:
+```yaml
+- a: banana
+  c: bananaC
+- a: apple
+  b: appleB2
+- a: dingo
+  c: dingoC
+```
+then
+```bash
+yq eval-all '(select(fi==1) | .[]) as $two | select(fi==0) | .[] |= (. as $cur |  $cur * ($two | select(.a == $cur.a)))' sample.yml another.yml
+```
+will output
+```yaml
+- a: apple
+  b: appleB2
+- a: kiwi
+  b: kiwiB
+- a: banana
+  b: bananaB
+  c: bananaC
+```
+
 ## Merge to prefix an element
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Style.md

@@ -1,4 +1,42 @@
 The style operator can be used to get or set the style of nodes (e.g. string style, yaml style)
+## Update and set style of a particular node (simple)
+Given a sample.yml file of:
+```yaml
+a:
+  b: thing
+  c: something
+```
+then
+```bash
+yq eval '.a.b = "new" | .a.b style="double"' sample.yml
+```
+will output
+```yaml
+a:
+  b: "new"
+  c: something
+```
+
+## Update and set style of a particular node using path variables
+You can use a variable to re-use a path
+
+Given a sample.yml file of:
+```yaml
+a:
+  b: thing
+  c: something
+```
+then
+```bash
+yq eval '.a.b as $x | $x = "new" | $x style="double"' sample.yml
+```
+will output
+```yaml
+a:
+  b: "new"
+  c: something
+```
+
 ## Set tagged style
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Traverse (Read).md

@@ -32,6 +32,21 @@ b: apple
 c: banana
 ```
 
+## Optional Splat
+Just like splat, but won't error if you run it against scalars
+
+Given a sample.yml file of:
+```yaml
+cat
+```
+then
+```bash
+yq eval '.[]' sample.yml
+```
+will output
+```yaml
+```
+
 ## Special characters
 Use quotes with brackets around path elements with special characters
 
@@ -98,6 +113,23 @@ will output
 null
 ```
 
+## Optional identifier
+Like jq, does not output an error when the yaml is not an array or object as expected
+
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+```
+then
+```bash
+yq eval '.a?' sample.yml
+```
+will output
+```yaml
+```
+
 ## Wildcard matching
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Unique.md

@@ -0,0 +1,82 @@
+This is used to filter out duplicated items in an array.
+
+## Unique array of scalars (string/numbers)
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+- 2
+```
+then
+```bash
+yq eval 'unique' sample.yml
+```
+will output
+```yaml
+- 1
+- 2
+- 3
+```
+
+## Unique nulls
+Unique works on the node value, so it considers different representations of nulls to be different
+
+Given a sample.yml file of:
+```yaml
+- ~
+- null
+- ~
+- null
+```
+then
+```bash
+yq eval 'unique' sample.yml
+```
+will output
+```yaml
+- ~
+- null
+```
+
+## Unique all nulls
+Run against the node tag to unique all the nulls
+
+Given a sample.yml file of:
+```yaml
+- ~
+- null
+- ~
+- null
+```
+then
+```bash
+yq eval 'unique_by(tag)' sample.yml
+```
+will output
+```yaml
+- ~
+```
+
+## Unique array object fields
+Given a sample.yml file of:
+```yaml
+- name: harry
+  pet: cat
+- name: billy
+  pet: dog
+- name: harry
+  pet: dog
+```
+then
+```bash
+yq eval 'unique_by(.name)' sample.yml
+```
+will output
+```yaml
+- name: harry
+  pet: cat
+- name: billy
+  pet: dog
+```
+

pkg/yqlib/doc/aa.md

@@ -4,7 +4,7 @@ In `yq` expressions are made up of operators and pipes. A context of nodes is pa
 
 Lets look at a couple of examples.
 
-## Example 1 - simple example
+## Example with a simple operator
 
 Given a document like:
 
@@ -37,10 +37,80 @@ This being the last operation in the expression, the results will be printed out
 3
 ```
 
-# Example 2 - operators with arguments.
+# Example with an operator that takes arguments.
+
+Given a document like:
+
+```yaml
+a: cat
+b: dog
+```
+
+with an expression:
+
+```
+.a = .b
+```
+
+The `=` operator takes two arguments, a `lhs` expression, which in this case is `.a` and `rhs` expression which is `.b`. 
+
+It pipes the current, lets call it 'root' context through the `lhs` expression of `.a` to return the node 
+
+```yaml
+cat
+```
+
+Note that this node holds not only its value 'cat', but comments and metadata too, including path and parent information.
+
+The `=` operator then pipes the 'root' context through the `rhs` expression of `.b` to return the node
+
+```yaml
+dog
+```
+
+Both sides have now been evaluated, so now the operator copies across the value from the RHS to the value on the LHS, and it returns the now updated context:
+
+```yaml
+a: dog
+b: dog
+```
+
+# Relative update (e.g. `|=`)
+There is another form of the `=` operator which we call the relative form. It's very similar to `=` but with one key difference when evaluating the RHS expression.
+
+In the plain form, we pass in the 'root' level context to the RHS expression. In relative form, we pass in _each result of the LHS_ to the RHS expression. Let's go through an example.
+
+Given a document like:
+
+```yaml
+a: 1
+b: thing
+```
+
+with an expression:
+
+```
+.a |= . + 1
+```
+
+Similar to the `=` operator, `|=` takes two operands, the LHS and RHS.
+
+It pipes the current context (the whole document) through the LHS expression  of `.a` to get the node value:
+
+```
+1
+```
+
+Now it pipes _that LHS context_ into the RHS expression `. + 1` (whereas in the `=` plain form it piped the original document context into the RHS) to yield:
 
 
-The `=` operator takes two arguments, a `lhs` expression and `rhs` expression. It runs the 'matching' nodes context against the `lhs` expression to find the nodes to update, lets call it `lhsNodes`, and then runs the matching nodes against the `rhs` to find the new values, lets call that `rhsNodes`. It updates the `lhsNodes` values with the `rhsNodes` values and _returns the original matching nodes_. This is important, where length changed the matching nodes to be new nodes with the length values, `=` returns the original matching nodes, albeit with some of the nodes values updated. So `.a = 3` will still return the parent matching node, but with the matching child updated.
+```
+2
+```
 
-Please see the individual operator docs for more information and examples.
+The assignment operator then copies across the value from the RHS to the value on the LHS, and it returns the now updated 'root' context:
 
+```yaml
+a: 2
+b: thing
+```

pkg/yqlib/doc/headers/Boolean Operators.md

@@ -1 +1,9 @@
-The `or` and `and` operators take two parameters and return a boolean result. `not` flips a boolean from true to false, or vice versa. These are most commonly used with the `select` operator to filter particular nodes.
+The `or` and `and` operators take two parameters and return a boolean result. 
+
+`not` flips a boolean from true to false, or vice versa. 
+
+`any` will return `true` if there are any `true` values in a array sequence, and `all` will return true if _all_ elements in an array are true.
+
+`any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet
+
+These are most commonly used with the `select` operator to filter particular nodes.

pkg/yqlib/doc/headers/Entries.md

@@ -0,0 +1 @@
+Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps.

pkg/yqlib/doc/headers/Unique.md

@@ -0,0 +1 @@
+This is used to filter out duplicated items in an array.

pkg/yqlib/expression_postfix.go

@@ -54,12 +54,26 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			opStack = opStack[0 : len(opStack)-1]
 			log.Debugf("deleteing open bracket from opstack")
 
-			//and append a collect to the opStack
+			//and append a collect to the result
-			result = append(result, &Operation{OperationType: collectOperator})
+			// hack - see if there's the optional traverse flag
+			// on the close op - move it to the collect op.
+			// allows for .["cat"]?
+			prefs := traversePreferences{}
+			closeTokenMatch := string(currentToken.Match.Bytes)
+			if closeTokenMatch[len(closeTokenMatch)-1:] == "?" {
+				prefs.OptionalTraverse = true
+			}
+			result = append(result, &Operation{OperationType: collectOperator, Preferences: prefs})
 			log.Debugf("put collect onto the result")
 			result = append(result, &Operation{OperationType: shortPipeOpType})
 			log.Debugf("put shortpipe onto the result")
 
+			//traverseArrayCollect is a sneaky op that needs to be included too
+			//when closing a []
+			if len(opStack) > 0 && opStack[len(opStack)-1].Operation != nil && opStack[len(opStack)-1].Operation.OperationType == traverseArrayOpType {
+				opStack, result = popOpToResult(opStack, result)
+			}
+
 		case closeBracket:
 			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != openBracket {
 				opStack, result = popOpToResult(opStack, result)

pkg/yqlib/expression_processing_test.go

@@ -12,6 +12,21 @@ var pathTests = []struct {
 	expectedTokens  []interface{}
 	expectedPostFix []interface{}
 }{
+	{
+		`.[0]`,
+		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
+		append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
+	{
+		`.[0][1]`,
+		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]", "TRAVERSE_ARRAY", "[", "1 (int64)", "]"),
+		append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "1 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
+	{
+		`"\""`,
+		append(make([]interface{}, 0), "\" (string)"),
+		append(make([]interface{}, 0), "\" (string)"),
+	},
 	{
 		`[]|join(".")`,
 		append(make([]interface{}, 0), "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")"),
@@ -62,6 +77,11 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"),
 		append(make([]interface{}, 0), "b", "a", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
 	},
+	{
+		`.b[.a]?`,
+		append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"),
+		append(make([]interface{}, 0), "b", "a", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
 	{
 		`.[]`,
 		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
@@ -72,6 +92,11 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
 		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
 	},
+	{
+		`.a[]?`,
+		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
+		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
 	{
 		`.a.[]`,
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
@@ -82,6 +107,11 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
 		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
 	},
+	{
+		`.a[0]?`,
+		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
+		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
 	{
 		`.a.[0]`,
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),

pkg/yqlib/expression_tokeniser.go

@@ -29,8 +29,9 @@ const (
 type token struct {
 	TokenType            tokenType
 	Operation            *Operation
-	AssignOperation      *Operation // e.g. tag (GetTag) op becomes AssignTag if '=' follows it
+	AssignOperation      *Operation      // e.g. tag (GetTag) op becomes AssignTag if '=' follows it
-	CheckForPostTraverse bool       // e.g. [1]cat should really be [1].cat
+	CheckForPostTraverse bool            // e.g. [1]cat should really be [1].cat
+	Match                *machines.Match // match that created this token
 
 }
 
@@ -63,12 +64,19 @@ func (t *token) toString(detail bool) string {
 func pathToken(wrapped bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		value := string(m.Bytes)
+		prefs := traversePreferences{}
+
+		if value[len(value)-1:] == "?" {
+			prefs.OptionalTraverse = true
+			value = value[:len(value)-1]
+		}
+
 		value = value[1:]
 		if wrapped {
 			value = unwrap(value)
 		}
 		log.Debug("PathToken %v", value)
-		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: traversePreferences{}}
+		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs}
 		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
 	}
 }
@@ -138,7 +146,7 @@ func assignAllCommentsOp(updateAssign bool) lex.Action {
 
 func literalToken(pType tokenType, checkForPost bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &token{TokenType: pType, CheckForPostTraverse: checkForPost}, nil
+		return &token{TokenType: pType, CheckForPostTraverse: checkForPost, Match: m}, nil
 	}
 }
 
@@ -181,6 +189,7 @@ func stringValue(wrapped bool) lex.Action {
 		if wrapped {
 			value = unwrap(value)
 		}
+		value = strings.ReplaceAll(value, "\\\"", "\"")
 		return &token{TokenType: operationToken, Operation: createValueOperation(value, value)}, nil
 	}
 }
@@ -251,6 +260,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType))
 	lexer.Add([]byte(`select`), opToken(selectOpType))
 	lexer.Add([]byte(`has`), opToken(hasOpType))
+	lexer.Add([]byte(`unique`), opToken(uniqueOpType))
+	lexer.Add([]byte(`unique_by`), opToken(uniqueByOpType))
 	lexer.Add([]byte(`explode`), opToken(explodeOpType))
 	lexer.Add([]byte(`or`), opToken(orOpType))
 	lexer.Add([]byte(`and`), opToken(andOpType))
@@ -266,6 +277,11 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`join`), opToken(joinStringOpType))
 	lexer.Add([]byte(`sub`), opToken(subStringOpType))
 
+	lexer.Add([]byte(`any`), opToken(anyOpType))
+	lexer.Add([]byte(`any_c`), opToken(anyConditionOpType))
+	lexer.Add([]byte(`all`), opToken(allOpType))
+	lexer.Add([]byte(`all_c`), opToken(allConditionOpType))
+
 	lexer.Add([]byte(`split`), opToken(splitStringOpType))
 	lexer.Add([]byte(`keys`), opToken(keysOpType))
 
@@ -278,6 +294,9 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`fileIndex`), opToken(getFileIndexOpType))
 	lexer.Add([]byte(`fi`), opToken(getFileIndexOpType))
 	lexer.Add([]byte(`path`), opToken(getPathOpType))
+	lexer.Add([]byte(`to_entries`), opToken(toEntriesOpType))
+	lexer.Add([]byte(`from_entries`), opToken(fromEntriesOpType))
+	lexer.Add([]byte(`with_entries`), opToken(withEntriesOpType))
 
 	lexer.Add([]byte(`lineComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
 
@@ -300,8 +319,8 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
 
-	lexer.Add([]byte(`\."[^ "]+"`), pathToken(true))
+	lexer.Add([]byte(`\."[^ "]+"\??`), pathToken(true))
-	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+`), pathToken(false))
+	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+\??`), pathToken(false))
 	lexer.Add([]byte(`\.`), selfToken())
 
 	lexer.Add([]byte(`\|`), opToken(pipeOpType))
@@ -316,12 +335,12 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`[Nn][Uu][Ll][Ll]`), nullValue())
 	lexer.Add([]byte(`~`), nullValue())
 
-	lexer.Add([]byte(`"[^"]*"`), stringValue(true))
+	lexer.Add([]byte(`"([^"\\]*(\\.[^"\\]*)*)"`), stringValue(true))
 	lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true))
 	lexer.Add([]byte(`env\([^\)]+\)`), envOp(false))
 
 	lexer.Add([]byte(`\[`), literalToken(openCollect, false))
-	lexer.Add([]byte(`\]`), literalToken(closeCollect, true))
+	lexer.Add([]byte(`\]\??`), literalToken(closeCollect, true))
 	lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
 	lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
 	lexer.Add([]byte(`\*[\+|\?d]*`), multiplyWithPrefs())
@@ -392,16 +411,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	skipNextToken = false
 	currentToken := tokens[index]
 
+	log.Debug("processing %v", currentToken.toString(true))
+
 	if currentToken.TokenType == traverseArrayCollect {
 		//need to put a traverse array then a collect currentToken
 		// do this by adding traverse then converting currentToken to collect
 
 		if index == 0 || tokens[index-1].TokenType != operationToken ||
 			tokens[index-1].Operation.OperationType != traversePathOpType {
+			log.Debug("  adding self")
 			op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"}
 			postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 		}
-
+		log.Debug("  adding traverse array")
 		op := &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 
@@ -412,17 +434,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	if index != len(tokens)-1 && currentToken.AssignOperation != nil &&
 		tokens[index+1].TokenType == operationToken &&
 		tokens[index+1].Operation.OperationType == assignOpType {
+		log.Debug("  its an update assign")
 		currentToken.Operation = currentToken.AssignOperation
 		currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign
 		skipNextToken = true
 	}
 
+	log.Debug("  adding token to the fixed list")
 	postProcessedTokens = append(postProcessedTokens, currentToken)
 
 	if index != len(tokens)-1 &&
 		((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) ||
 			(currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) {
-
+		log.Debug("  adding empty")
 		op := &Operation{OperationType: emptyOpType, StringValue: "EMPTY"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}
@@ -430,12 +454,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
 		tokens[index+1].TokenType == operationToken &&
 		tokens[index+1].Operation.OperationType == traversePathOpType {
+		log.Debug("  adding pipe because the next thing is traverse")
 		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}
 	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
 		tokens[index+1].TokenType == openCollect {
 
+		// if tokens[index].TokenType == closeCollect {
+		// 	log.Debug("  adding pipe because next is opencollect")
+		// 	op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
+		// 	postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+		// }
+		log.Debug("  adding traverArray because next is opencollect")
 		op := &Operation{OperationType: traverseArrayOpType}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}

pkg/yqlib/lib.go

@@ -20,9 +20,6 @@ type operationType struct {
 	Handler    operatorHandler
 }
 
-// operators TODO:
-// - mergeEmpty (sets only if the document is empty, do I do that now?)
-
 var orOpType = &operationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: orOperator}
 var andOpType = &operationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: andOperator}
 var reduceOpType = &operationType{Type: "REDUCE", NumArgs: 2, Precedence: 35, Handler: reduceOperator}
@@ -60,6 +57,16 @@ var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence:
 
 var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: collectOperator}
+
+var anyOpType = &operationType{Type: "ANY", NumArgs: 0, Precedence: 50, Handler: anyOperator}
+var allOpType = &operationType{Type: "ALL", NumArgs: 0, Precedence: 50, Handler: allOperator}
+var anyConditionOpType = &operationType{Type: "ANY_CONDITION", NumArgs: 1, Precedence: 50, Handler: anyOperator}
+var allConditionOpType = &operationType{Type: "ALL_CONDITION", NumArgs: 1, Precedence: 50, Handler: allOperator}
+
+var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: 50, Handler: toEntriesOperator}
+var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator}
+var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator}
+
 var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator}
 var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator}
 var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}
@@ -94,6 +101,8 @@ var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs:
 
 var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator}
 var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator}
+var uniqueOpType = &operationType{Type: "UNIQUE", NumArgs: 0, Precedence: 50, Handler: unique}
+var uniqueByOpType = &operationType{Type: "UNIQUE_BY", NumArgs: 1, Precedence: 50, Handler: uniqueBy}
 var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
 
 type Operation struct {

pkg/yqlib/operator_add.go

@@ -39,7 +39,7 @@ func toNodes(candidate *CandidateNode) []*yaml.Node {
 func addOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("Add operator")
 
-	return crossFunction(d, context, expressionNode, add, false)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, add, false)
 }
 
 func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

pkg/yqlib/operator_add_test.go

@@ -14,6 +14,22 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[1 a], (!!int)::3\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: "(.a + .b) as $x",
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: 0`,
+		expression: ".a += .b.c",
+		expected: []string{
+			"D0, P[], (doc)::a: 0\n",
+		},
+	},
 	{
 		description: "Concatenate and assign arrays",
 		document:    `{a: {val: thing, b: [cat,dog]}}`,

pkg/yqlib/operator_alternative.go

@@ -2,7 +2,7 @@ package yqlib
 
 func alternativeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- alternative")
-	return crossFunction(d, context, expressionNode, alternativeFunc, true)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, alternativeFunc, true)
 }
 
 func alternativeFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

pkg/yqlib/operator_alternative_test.go

@@ -5,6 +5,20 @@ import (
 )
 
 var alternativeOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		expression: `.b // .c`,
+		document:   `a: bridge`,
+		expected:   []string{},
+	},
+	{
+		skipDoc:    true,
+		expression: `(.b // "hello") as $x`,
+		document:   `a: bridge`,
+		expected: []string{
+			"D0, P[], (doc)::a: bridge\n",
+		},
+	},
 	{
 		description: "LHS is defined",
 		expression:  `.a // "hello"`,

pkg/yqlib/operator_anchors_aliases.go

@@ -12,7 +12,7 @@ func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *
 
 	aliasName := ""
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -32,7 +32,7 @@ func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *
 		log.Debugf("Setting aliasName : %v", candidate.GetKey())
 
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}
@@ -41,8 +41,10 @@ func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *
 			}
 		}
 
-		candidate.Node.Kind = yaml.AliasNode
+		if aliasName != "" {
-		candidate.Node.Value = aliasName
+			candidate.Node.Kind = yaml.AliasNode
+			candidate.Node.Value = aliasName
+		}
 	}
 	return context, nil
 }
@@ -66,7 +68,7 @@ func assignAnchorOperator(d *dataTreeNavigator, context Context, expressionNode
 
 	anchorName := ""
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -87,7 +89,7 @@ func assignAnchorOperator(d *dataTreeNavigator, context Context, expressionNode
 		log.Debugf("Setting anchorName of : %v", candidate.GetKey())
 
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}
@@ -176,7 +178,7 @@ func explodeNode(node *yaml.Node, context Context) error {
 			} else {
 				if valueNode.Kind == yaml.SequenceNode {
 					log.Debugf("an alias merge list!")
-					for index := 0; index < len(valueNode.Content); index = index + 1 {
+					for index := len(valueNode.Content) - 1; index >= 0; index = index - 1 {
 						aliasNode := valueNode.Content[index]
 						err := applyAlias(node, aliasNode.Alias, index, context.ChildContext(newContent))
 						if err != nil {

pkg/yqlib/operator_anchors_aliases_test.go

@@ -4,7 +4,36 @@ import (
 	"testing"
 )
 
+var specDocument = `- &CENTER { x: 1, y: 2 }
+- &LEFT { x: 0, y: 2 }
+- &BIG { r: 10 }
+- &SMALL { r: 1 }
+`
+
+var expectedSpecResult = "D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n"
+
 var anchorOperatorScenarios = []expressionScenario{
+	{
+		description:    "Merge one map",
+		subdescription: "see https://yaml.org/type/merge.html",
+		document:       specDocument + "- << : *CENTER\n  r: 10\n",
+		expression:     ".[4] | explode(.)",
+		expected:       []string{expectedSpecResult},
+	},
+	{
+		description:    "Merge multiple maps",
+		subdescription: "see https://yaml.org/type/merge.html",
+		document:       specDocument + "- << : [ *CENTER, *BIG ]\n",
+		expression:     ".[4] | explode(.)",
+		expected:       []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"},
+	},
+	{
+		description:    "Override",
+		subdescription: "see https://yaml.org/type/merge.html",
+		document:       specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n  x: 1\n",
+		expression:     ".[4] | explode(.)",
+		expected:       []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"},
+	},
 	{
 		description: "Get anchor",
 		document:    `a: &billyBob cat`,
@@ -29,6 +58,22 @@ var anchorOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: &cat {b: cat}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `a: {c: cat}`,
+		expression: `.a anchor |= .b`,
+		expected: []string{
+			"D0, P[], (doc)::a: {c: cat}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: {c: cat}`,
+		expression: `.a anchor = .b`,
+		expected: []string{
+			"D0, P[], (doc)::a: {c: cat}\n",
+		},
+	},
 	{
 		description: "Get alias",
 		document:    `{b: &billyBob meow, a: *billyBob}`,
@@ -45,6 +90,30 @@ var anchorOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{b: &meow purr, a: *meow}\n",
 		},
 	},
+	{
+		description: "Set alias to blank does nothing",
+		document:    `{b: &meow purr, a: cat}`,
+		expression:  `.a alias = ""`,
+		expected: []string{
+			"D0, P[], (doc)::{b: &meow purr, a: cat}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{b: &meow purr, a: cat}`,
+		expression: `.a alias = .c`,
+		expected: []string{
+			"D0, P[], (doc)::{b: &meow purr, a: cat}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{b: &meow purr, a: cat}`,
+		expression: `.a alias |= .c`,
+		expected: []string{
+			"D0, P[], (doc)::{b: &meow purr, a: cat}\n",
+		},
+	},
 	{
 		description: "Set alias relatively using assign-update",
 		document:    `{b: &meow purr, a: {f: meow}}`,
@@ -91,9 +160,9 @@ bar:
     c: bar_c
 foobarList:
     b: bar_b
-    a: foo_a
+    thing: foo_thing
-    thing: bar_thing
     c: foobarList_c
+    a: foo_a
 foobar:
     c: foo_c
     a: foo_a
@@ -106,7 +175,7 @@ foobar:
 		expression: `.foo* | explode(.) | (. style="flow")`,
 		expected: []string{
 			"D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n",
-			"D0, P[foobarList], (!!map)::{b: bar_b, a: foo_a, thing: bar_thing, c: foobarList_c}\n",
+			"D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n",
 			"D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n",
 		},
 	},
@@ -116,7 +185,7 @@ foobar:
 		expression: `.foo* | explode(explode(.)) | (. style="flow")`,
 		expected: []string{
 			"D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n",
-			"D0, P[foobarList], (!!map)::{b: bar_b, a: foo_a, thing: bar_thing, c: foobarList_c}\n",
+			"D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n",
 			"D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n",
 		},
 	},
@@ -130,7 +199,7 @@ foobar:
 	},
 }
 
-func TestAnchorAliaseOperatorScenarios(t *testing.T) {
+func TestAnchorAliasOperatorScenarios(t *testing.T) {
 	for _, tt := range anchorOperatorScenarios {
 		testScenario(t, &tt)
 	}

pkg/yqlib/operator_assign.go

@@ -7,7 +7,7 @@ func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode
 	}
 	var rhs Context
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err = d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err = d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 	}
 
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
@@ -44,7 +44,7 @@ func assignAttributesOperator(d *dataTreeNavigator, context Context, expressionN
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 
 		if err != nil {
 			return Context{}, err

pkg/yqlib/operator_assign_test.go

@@ -12,6 +12,22 @@ var assignOperatorScenarios = []expressionScenario{
 			"D0, P[], ()::a:\n    b: cat\nx: frog\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: `.a |= .b`,
+		expected: []string{
+			"D0, P[], (doc)::{a: null}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: `.a = .b`,
+		expected: []string{
+			"D0, P[], (doc)::{a: null}\n",
+		},
+	},
 	{
 		description: "Update node to be the child value",
 		document:    `{a: {b: {g: foof}}}`,

pkg/yqlib/operator_booleans.go

@@ -2,14 +2,13 @@ package yqlib
 
 import (
 	"container/list"
+	"fmt"
 
 	yaml "gopkg.in/yaml.v3"
 )
 
-func isTruthy(c *CandidateNode) (bool, error) {
+func isTruthyNode(node *yaml.Node) (bool, error) {
-	node := unwrapDoc(c.Node)
 	value := true
-
 	if node.Tag == "!!null" {
 		return false, nil
 	}
@@ -23,41 +22,131 @@ func isTruthy(c *CandidateNode) (bool, error) {
 	return value, nil
 }
 
+func isTruthy(c *CandidateNode) (bool, error) {
+	node := unwrapDoc(c.Node)
+	return isTruthyNode(node)
+}
+
 type boolOp func(bool, bool) bool
 
 func performBoolOp(op boolOp) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-		lhs.Node = unwrapDoc(lhs.Node)
+		owner := lhs
-		rhs.Node = unwrapDoc(rhs.Node)
 
-		lhsTrue, errDecoding := isTruthy(lhs)
+		if lhs == nil && rhs == nil {
-		if errDecoding != nil {
+			owner = &CandidateNode{}
-			return nil, errDecoding
+		} else if lhs == nil {
+			owner = rhs
 		}
 
-		rhsTrue, errDecoding := isTruthy(rhs)
+		var errDecoding error
-		if errDecoding != nil {
+		lhsTrue := false
-			return nil, errDecoding
+		if lhs != nil {
-		}
+			lhs.Node = unwrapDoc(lhs.Node)
+			lhsTrue, errDecoding = isTruthy(lhs)
 
-		return createBooleanCandidate(lhs, op(lhsTrue, rhsTrue)), nil
+			if errDecoding != nil {
+				return nil, errDecoding
+			}
+		}
+		log.Debugf("-- lhsTrue", lhsTrue)
+
+		rhsTrue := false
+		if rhs != nil {
+			rhs.Node = unwrapDoc(rhs.Node)
+			rhsTrue, errDecoding = isTruthy(rhs)
+			if errDecoding != nil {
+				return nil, errDecoding
+			}
+		}
+		log.Debugf("-- rhsTrue", rhsTrue)
+
+		return createBooleanCandidate(owner, op(lhsTrue, rhsTrue)), nil
 	}
 }
 
+func findBoolean(wantBool bool, d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, sequenceNode *yaml.Node) (bool, error) {
+	for _, node := range sequenceNode.Content {
+
+		if expressionNode != nil {
+			//need to evaluate the expression against the node
+			candidate := &CandidateNode{Node: node}
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode)
+			if err != nil {
+				return false, err
+			}
+			if rhs.MatchingNodes.Len() > 0 {
+				node = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node
+			} else {
+				// no results found, ignore this entry
+				continue
+			}
+		}
+
+		truthy, err := isTruthyNode(node)
+		if err != nil {
+			return false, err
+		}
+		if truthy == wantBool {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func allOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+		if candidateNode.Kind != yaml.SequenceNode {
+			return Context{}, fmt.Errorf("any only supports arrays, was %v", candidateNode.Tag)
+		}
+		booleanResult, err := findBoolean(false, d, context, expressionNode.Rhs, candidateNode)
+		if err != nil {
+			return Context{}, err
+		}
+		result := createBooleanCandidate(candidate, !booleanResult)
+		results.PushBack(result)
+	}
+	return context.ChildContext(results), nil
+}
+
+func anyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+		if candidateNode.Kind != yaml.SequenceNode {
+			return Context{}, fmt.Errorf("any only supports arrays, was %v", candidateNode.Tag)
+		}
+		booleanResult, err := findBoolean(true, d, context, expressionNode.Rhs, candidateNode)
+		if err != nil {
+			return Context{}, err
+		}
+		result := createBooleanCandidate(candidate, booleanResult)
+		results.PushBack(result)
+	}
+	return context.ChildContext(results), nil
+}
+
 func orOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- orOp")
-	return crossFunction(d, context, expressionNode, performBoolOp(
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, performBoolOp(
 		func(b1 bool, b2 bool) bool {
+			log.Debugf("-- peformingOrOp with %v and %v", b1, b2)
 			return b1 || b2
-		}), false)
+		}), true)
 }
 
 func andOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- AndOp")
-	return crossFunction(d, context, expressionNode, performBoolOp(
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, performBoolOp(
 		func(b1 bool, b2 bool) bool {
 			return b1 && b2
-		}), false)
+		}), true)
 }
 
 func notOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {

pkg/yqlib/operator_booleans_test.go

@@ -6,14 +6,38 @@ import (
 
 var booleanOperatorScenarios = []expressionScenario{
 	{
-		description: "OR example",
+		description: "`or` example",
 		expression:  `true or false`,
 		expected: []string{
 			"D0, P[], (!!bool)::true\n",
 		},
 	},
 	{
-		description: "AND example",
+		skipDoc:    true,
+		document:   "b: hi",
+		expression: `.a or .c`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "b: hi",
+		expression: `select(.a or .b)`,
+		expected: []string{
+			"D0, P[], (doc)::b: hi\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "b: hi",
+		expression: `select((.a and .b) | not)`,
+		expected: []string{
+			"D0, P[], (doc)::b: hi\n",
+		},
+	},
+	{
+		description: "`and` example",
 		expression:  `true and false`,
 		expected: []string{
 			"D0, P[], (!!bool)::false\n",
@@ -27,6 +51,86 @@ var booleanOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::- {a: bird, b: dog}\n- {a: cat, b: fly}\n",
 		},
 	},
+	{
+		description: "`any` returns true if any boolean in a given array is true",
+		document:    `[false, true]`,
+		expression:  "any",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		description: "`any` returns false for an empty array",
+		document:    `[]`,
+		expression:  "any",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "`any_c` returns true if any element in the array is true for the given condition.",
+		document:    "a: [rad, awesome]\nb: [meh, whatever]",
+		expression:  `.[] |= any_c(. == "awesome")`,
+		expected: []string{
+			"D0, P[], (doc)::a: true\nb: false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{pet: cat}]`,
+		expression: `any_c(.name == "harry") as $c`,
+		expected: []string{
+			"D0, P[], (doc)::[{pet: cat}]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{pet: cat}]`,
+		expression: `all_c(.name == "harry") as $c`,
+		expected: []string{
+			"D0, P[], (doc)::[{pet: cat}]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[false, false]`,
+		expression: "any",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "`all` returns true if all booleans in a given array are true",
+		document:    `[true, true]`,
+		expression:  "all",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[false, true]`,
+		expression: "all",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "`all` returns true for an empty array",
+		document:    `[]`,
+		expression:  "all",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		description: "`all_c` returns true if all elements in the array are true for the given condition.",
+		document:    "a: [rad, awesome]\nb: [meh, 12]",
+		expression:  `.[] |= all_c(tag == "!!str")`,
+		expected: []string{
+			"D0, P[], (doc)::a: true\nb: false\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		expression: `false or false`,
@@ -45,6 +149,22 @@ var booleanOperatorScenarios = []expressionScenario{
 			"D0, P[b], (!!bool)::true\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: `(.a.b or .c) as $x`,
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: `(.a.b and .c) as $x`,
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
 	{
 		description: "Not true is false",
 		expression:  `true | not`,

pkg/yqlib/operator_collect_object.go

@@ -17,9 +17,12 @@ import (
 ...
 */
 
-func collectObjectOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+func collectObjectOperator(d *dataTreeNavigator, originalContext Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- collectObjectOperation")
 
+	context := originalContext.Clone()
+	context.DontAutoCreate = false
+
 	if context.MatchingNodes.Len() == 0 {
 		node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map", Value: "{}"}
 		candidate := &CandidateNode{Node: node}

pkg/yqlib/operator_collect_object_test.go

@@ -5,6 +5,30 @@ import (
 )
 
 var collectObjectOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   "a: []",
+		expression: `.a += [{"key": "att2", "value": "val2"}]`,
+		expected: []string{
+			"D0, P[], (doc)::a: [{key: att2, value: val2}]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "",
+		expression: `.a += {"key": "att2", "value": "val2"}`,
+		expected: []string{
+			"D0, P[], ()::a:\n    key: att2\n    value: val2\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "",
+		expression: `.a += [0]`,
+		expected: []string{
+			"D0, P[], ()::a:\n    - 0\n",
+		},
+	},
 	{
 		description: `Collect empty object`,
 		document:    ``,

pkg/yqlib/operator_collect_test.go

@@ -5,6 +5,14 @@ import (
 )
 
 var collectOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   ``,
+		expression: `.a += [0]`,
+		expected: []string{
+			"D0, P[], ()::a:\n    - 0\n",
+		},
+	},
 	{
 		description: "Collect empty",
 		document:    ``,

pkg/yqlib/operator_comments.go

@@ -27,7 +27,7 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
 
 	comment := ""
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -41,7 +41,7 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
 		candidate := el.Value.(*CandidateNode)
 
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}

pkg/yqlib/operator_comments_test.go

@@ -62,6 +62,22 @@ var commentOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: cat\n\n# cat\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `a: cat`,
+		expression: `. footComment=.b.d`,
+		expected: []string{
+			"D0, P[], (doc)::a: cat\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: cat`,
+		expression: `. footComment|=.b.d`,
+		expected: []string{
+			"D0, P[], (doc)::a: cat\n",
+		},
+	},
 	{
 		description: "Remove comment",
 		document:    "a: cat # comment\nb: dog # leave this",
@@ -71,7 +87,7 @@ var commentOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description:    "Remove all comments",
+		description:    "Remove (strip) all comments",
 		subdescription: "Note the use of `...` to ensure key nodes are included.",
 		document:       "# hi\n\na: cat # comment\n\n# great\n\nb: # key comment",
 		expression:     `... comments=""`,

pkg/yqlib/operator_delete.go

@@ -7,9 +7,7 @@ import (
 )
 
 func deleteChildOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	contextToUse := context.Clone()
+	nodesToDelete, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
-	contextToUse.DontAutoCreate = true
-	nodesToDelete, err := d.GetMatchingNodes(contextToUse, expressionNode.Rhs)
 
 	if err != nil {
 		return Context{}, err

pkg/yqlib/operator_entries.go

@@ -0,0 +1,162 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func entrySeqFor(key *yaml.Node, value *yaml.Node) *yaml.Node {
+	var keyKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "key"}
+	var valueKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "value"}
+
+	return &yaml.Node{
+		Kind:    yaml.MappingNode,
+		Tag:     "!!map",
+		Content: []*yaml.Node{keyKey, key, valueKey, value},
+	}
+}
+
+func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {
+	var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+	var entriesNode = candidateNode.CreateChild(nil, sequence)
+
+	var contents = unwrapDoc(candidateNode.Node).Content
+	for index := 0; index < len(contents); index = index + 2 {
+		key := contents[index]
+		value := contents[index+1]
+
+		sequence.Content = append(sequence.Content, entrySeqFor(key, value))
+	}
+	return entriesNode
+}
+
+func toEntriesfromSeq(candidateNode *CandidateNode) *CandidateNode {
+	var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+	var entriesNode = candidateNode.CreateChild(nil, sequence)
+
+	var contents = unwrapDoc(candidateNode.Node).Content
+	for index := 0; index < len(contents); index = index + 1 {
+		key := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!int", Value: fmt.Sprintf("%v", index)}
+		value := contents[index]
+
+		sequence.Content = append(sequence.Content, entrySeqFor(key, value))
+	}
+	return entriesNode
+}
+
+func toEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	var results = list.New()
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+
+		switch candidateNode.Kind {
+		case yaml.MappingNode:
+			results.PushBack(toEntriesFromMap(candidate))
+
+		case yaml.SequenceNode:
+			results.PushBack(toEntriesfromSeq(candidate))
+		default:
+			if candidateNode.Tag != "!!null" {
+				return Context{}, fmt.Errorf("%v has no keys", candidate.Node.Tag)
+			}
+		}
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func parseEntry(d *dataTreeNavigator, entry *yaml.Node, position int) (*yaml.Node, *yaml.Node, error) {
+	prefs := traversePreferences{DontAutoCreate: true}
+	candidateNode := &CandidateNode{Node: entry}
+
+	keyResults, err := traverseMap(Context{}, candidateNode, "key", prefs, false)
+
+	if err != nil {
+		return nil, nil, err
+	} else if keyResults.Len() != 1 {
+		return nil, nil, fmt.Errorf("Expected to find one 'key' entry but found %v in position %v", keyResults.Len(), position)
+	}
+
+	valueResults, err := traverseMap(Context{}, candidateNode, "value", prefs, false)
+
+	if err != nil {
+		return nil, nil, err
+	} else if valueResults.Len() != 1 {
+		return nil, nil, fmt.Errorf("Expected to find one 'value' entry but found %v in position %v", valueResults.Len(), position)
+	}
+
+	return keyResults.Front().Value.(*CandidateNode).Node, valueResults.Front().Value.(*CandidateNode).Node, nil
+
+}
+
+func fromEntries(d *dataTreeNavigator, candidateNode *CandidateNode) (*CandidateNode, error) {
+	var node = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
+	var mapCandidateNode = candidateNode.CreateChild(nil, node)
+
+	var contents = unwrapDoc(candidateNode.Node).Content
+
+	for index := 0; index < len(contents); index = index + 1 {
+		key, value, err := parseEntry(d, contents[index], index)
+		if err != nil {
+			return nil, err
+		}
+
+		node.Content = append(node.Content, key, value)
+	}
+	return mapCandidateNode, nil
+}
+
+func fromEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	var results = list.New()
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+
+		switch candidateNode.Kind {
+		case yaml.SequenceNode:
+			mapResult, err := fromEntries(d, candidate)
+			if err != nil {
+				return Context{}, err
+			}
+			results.PushBack(mapResult)
+		default:
+			return Context{}, fmt.Errorf("from entries only runs against arrays")
+		}
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	//to_entries on the context
+	toEntries, err := toEntriesOperator(d, context, expressionNode)
+	if err != nil {
+		return Context{}, err
+	}
+
+	//run expression against entries
+	// splat toEntries and pipe it into Rhs
+	splatted, err := splat(d, toEntries, traversePreferences{})
+	if err != nil {
+		return Context{}, err
+	}
+
+	result, err := d.GetMatchingNodes(splatted, expressionNode.Rhs)
+	log.Debug("expressionNode.Rhs %v", expressionNode.Rhs.Operation.OperationType)
+	log.Debug("result %v", result)
+	if err != nil {
+		return Context{}, err
+	}
+
+	collected, err := collectOperator(d, result, expressionNode)
+	if err != nil {
+		return Context{}, err
+	}
+
+	//from_entries on the result
+	return fromEntriesOperator(d, collected, expressionNode)
+}

pkg/yqlib/operator_entries_test.go

@@ -0,0 +1,62 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var entriesOperatorScenarios = []expressionScenario{
+	{
+		description: "to_entries Map",
+		document:    `{a: 1, b: 2}`,
+		expression:  `to_entries`,
+		expected: []string{
+			"D0, P[], (!!seq)::- key: a\n  value: 1\n- key: b\n  value: 2\n",
+		},
+	},
+	{
+		description: "to_entries Array",
+		document:    `[a, b]`,
+		expression:  `to_entries`,
+		expected: []string{
+			"D0, P[], (!!seq)::- key: 0\n  value: a\n- key: 1\n  value: b\n",
+		},
+	},
+	{
+		description: "to_entries null",
+		document:    `null`,
+		expression:  `to_entries`,
+		expected:    []string{},
+	},
+	{
+		description: "from_entries map",
+		document:    `{a: 1, b: 2}`,
+		expression:  `to_entries | from_entries`,
+		expected: []string{
+			"D0, P[], (!!map)::a: 1\nb: 2\n",
+		},
+	},
+	{
+		description:    "from_entries with numeric key indexes",
+		subdescription: "from_entries always creates a map, even for numeric keys",
+		document:       `[a,b]`,
+		expression:     `to_entries | from_entries`,
+		expected: []string{
+			"D0, P[], (!!map)::0: a\n1: b\n",
+		},
+	},
+	{
+		description: "Use with_entries to update keys",
+		document:    `{a: 1, b: 2}`,
+		expression:  `with_entries(.key |= "KEY_" + .)`,
+		expected: []string{
+			"D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n",
+		},
+	},
+}
+
+func TestEntriesOperatorScenarios(t *testing.T) {
+	for _, tt := range entriesOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Entries", entriesOperatorScenarios)
+}

pkg/yqlib/operator_equals.go

@@ -4,12 +4,33 @@ import "gopkg.in/yaml.v3"
 
 func equalsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- equalsOperation")
-	return crossFunction(d, context, expressionNode, isEquals(false), false)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, isEquals(false), true)
 }
 
 func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		value := false
+		log.Debugf("-- isEquals cross function")
+		if lhs == nil && rhs == nil {
+			owner := &CandidateNode{}
+			return createBooleanCandidate(owner, !flip), nil
+		} else if lhs == nil {
+			log.Debugf("lhs nil, but rhs is not")
+			rhsNode := unwrapDoc(rhs.Node)
+			value := rhsNode.Tag == "!!null"
+			if flip {
+				value = !value
+			}
+			return createBooleanCandidate(rhs, value), nil
+		} else if rhs == nil {
+			log.Debugf("lhs not nil, but rhs is")
+			lhsNode := unwrapDoc(lhs.Node)
+			value := lhsNode.Tag == "!!null"
+			if flip {
+				value = !value
+			}
+			return createBooleanCandidate(lhs, value), nil
+		}
 
 		lhsNode := unwrapDoc(lhs.Node)
 		rhsNode := unwrapDoc(rhs.Node)
@@ -28,6 +49,6 @@ func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *Candid
 }
 
 func notEqualsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	log.Debugf("-- equalsOperation")
+	log.Debugf("-- notEqualsOperator")
-	return crossFunction(d, context, expressionNode, isEquals(true), false)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, isEquals(true), true)
 }

pkg/yqlib/operator_equals_test.go

@@ -5,6 +5,13 @@ import (
 )
 
 var equalsOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		expression: ".a == .b",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "cat",
@@ -14,6 +21,60 @@ var equalsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::false\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: "(.a == .b) as $x",
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: ".a == .b",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: "(.a != .b) as $x",
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: ".a != .b",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{a: {b: 10}}",
+		expression: "select(.c != null)",
+		expected:   []string{},
+	},
+	{
+		skipDoc:    true,
+		document:   "{a: {b: 10}}",
+		expression: "select(.d == .c)",
+		expected: []string{
+			"D0, P[], (doc)::{a: {b: 10}}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{a: {b: 10}}",
+		expression: "select(null == .c)",
+		expected: []string{
+			"D0, P[], (doc)::{a: {b: 10}}\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "{a: { b: {things: \"\"}, f: [1], g: [] }}",
@@ -87,6 +148,22 @@ var equalsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::true\n",
 		},
 	},
+	{
+		description: "Non exisitant key doesn't equal a value",
+		document:    "a: frog",
+		expression:  `select(.b != "thing")`,
+		expected: []string{
+			"D0, P[], (doc)::a: frog\n",
+		},
+	},
+	{
+		description: "Two non existant keys are equal",
+		document:    "a: frog",
+		expression:  `select(.b == .c)`,
+		expected: []string{
+			"D0, P[], (doc)::a: frog\n",
+		},
+	},
 }
 
 func TestEqualOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_has.go

@@ -12,14 +12,19 @@ func hasOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	log.Debugf("-- hasOperation")
 	var results = list.New()
 
-	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
-	wanted := rhs.MatchingNodes.Front().Value.(*CandidateNode).Node
-	wantedKey := wanted.Value
 
 	if err != nil {
 		return Context{}, err
 	}
 
+	wantedKey := "null"
+	wanted := &yaml.Node{Tag: "!!null"}
+	if rhs.MatchingNodes.Len() != 0 {
+		wanted = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node
+		wantedKey = wanted.Value
+	}
+
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 

pkg/yqlib/operator_has_test.go

@@ -13,6 +13,22 @@ var hasOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::true\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `a: hello`,
+		expression: `has(.b) as $c`,
+		expected: []string{
+			"D0, P[], (doc)::a: hello\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: hello`,
+		expression: `has(.b)`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
 	{
 		description: "Has map key",
 		document: `- a: "yes"

pkg/yqlib/operator_multiply_test.go

@@ -24,6 +24,12 @@ list2:
     - "123"
 `
 
+var mergeArraysObjectKeysText = `It's a complex command, the trickyness comes from needing to have the right context in the expressions.
+First we save the second array into a variable '$two' which lets us reference it later.
+We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. 
+We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter.
+`
+
 var multiplyOperatorScenarios = []expressionScenario{
 	{
 		description: "Multiply integers",
@@ -245,6 +251,16 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!seq)::[{name: fred, age: 34}, {name: bob, age: 32}]\n",
 		},
 	},
+	{
+		description:    "Merge arrays of objects together, matching on a key",
+		subdescription: mergeArraysObjectKeysText,
+		document:       `[{a: apple, b: appleB}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB}]`,
+		document2:      `[{a: banana, c: bananaC}, {a: apple, b: appleB2}, {a: dingo, c: dingoC}]`,
+		expression:     `(select(fi==1) | .[]) as $two | select(fi==0) | .[] |= (. as $cur |  $cur * ($two | select(.a == $cur.a)))`,
+		expected: []string{
+			"D0, P[], (doc)::[{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}]\n",
+		},
+	},
 	{
 		description: "Merge to prefix an element",
 		document:    `{a: cat, b: dog}`,

pkg/yqlib/operator_select.go

@@ -11,9 +11,7 @@ func selectOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		childContext := context.SingleChildContext(candidate)
+		rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
-		childContext.DontAutoCreate = true
-		rhs, err := d.GetMatchingNodes(childContext, expressionNode.Rhs)
 
 		if err != nil {
 			return Context{}, err
@@ -24,7 +22,9 @@ func selectOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 
 		if first != nil {
 			result := first.Value.(*CandidateNode)
+			log.Debugf("result %v", NodeToString(result))
 			includeResult, errDecoding := isTruthy(result)
+			log.Debugf("isTruthy %v", includeResult)
 			if errDecoding != nil {
 				return Context{}, errDecoding
 			}

pkg/yqlib/operator_select_test.go

@@ -14,6 +14,16 @@ var selectOperatorScenarios = []expressionScenario{
 			"D0, P[1], (!!str)::goat\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "a: hello",
+		document2:  "b: world",
+		expression: `select(.a == "hello" or .b == "world")`,
+		expected: []string{
+			"D0, P[], (doc)::a: hello\n",
+			"D0, P[], (doc)::b: world\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `[{animal: cat, legs: {cool: true}}, {animal: fish}]`,

pkg/yqlib/operator_sort_keys.go

@@ -10,7 +10,7 @@ func sortKeysOperator(d *dataTreeNavigator, context Context, expressionNode *Exp
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}

pkg/yqlib/operator_sort_keys_test.go

@@ -13,6 +13,14 @@ var sortKeysOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: blah, b: bing, c: frog}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{c: frog}`,
+		expression: `sortKeys(.d)`,
+		expected: []string{
+			"D0, P[], (doc)::{c: frog}\n",
+		},
+	},
 	{
 		description:    "Sort keys recursively",
 		subdescription: "Note the array elements are left unsorted, but maps inside arrays are sorted",

pkg/yqlib/operator_strings.go

@@ -13,7 +13,7 @@ func getSubstituteParameters(d *dataTreeNavigator, block *ExpressionNode, contex
 	regEx := ""
 	replacementText := ""
 
-	regExNodes, err := d.GetMatchingNodes(context, block.Lhs)
+	regExNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), block.Lhs)
 	if err != nil {
 		return "", "", err
 	}
@@ -78,7 +78,7 @@ func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *E
 	log.Debugf("-- joinStringOperator")
 	joinStr := ""
 
-	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 	if err != nil {
 		return Context{}, err
 	}
@@ -119,7 +119,7 @@ func splitStringOperator(d *dataTreeNavigator, context Context, expressionNode *
 	log.Debugf("-- splitStringOperator")
 	splitStr := ""
 
-	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 	if err != nil {
 		return Context{}, err
 	}

pkg/yqlib/operator_style.go

@@ -31,7 +31,7 @@ func assignStyleOperator(d *dataTreeNavigator, context Context, expressionNode *
 	log.Debugf("AssignStyleOperator: %v")
 	var style yaml.Style
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -54,7 +54,7 @@ func assignStyleOperator(d *dataTreeNavigator, context Context, expressionNode *
 		candidate := el.Value.(*CandidateNode)
 		log.Debugf("Setting style of : %v", candidate.GetKey())
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}

pkg/yqlib/operator_style_test.go

@@ -5,6 +5,23 @@ import (
 )
 
 var styleOperatorScenarios = []expressionScenario{
+	{
+		description: "Update and set style of a particular node (simple)",
+		document:    `a: {b: thing, c: something}`,
+		expression:  `.a.b = "new" | .a.b style="double"`,
+		expected: []string{
+			"D0, P[], (doc)::a: {b: \"new\", c: something}\n",
+		},
+	},
+	{
+		description:    "Update and set style of a particular node using path variables",
+		subdescription: "You can use a variable to re-use a path",
+		document:       `a: {b: thing, c: something}`,
+		expression:     `.a.b as $x | $x = "new" | $x style="double"`,
+		expected: []string{
+			"D0, P[], (doc)::a: {b: \"new\", c: something}\n",
+		},
+	},
 	{
 		description: "Set tagged style",
 		document:    `{a: cat, b: 5, c: 3.2, e: true}`,

pkg/yqlib/operator_subtract.go

@@ -25,7 +25,7 @@ func subtractAssignOperator(d *dataTreeNavigator, context Context, expressionNod
 func subtractOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("Subtract operator")
 
-	return crossFunction(d, context, expressionNode, subtract, false)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, subtract, false)
 }
 
 func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

pkg/yqlib/operator_subtract_test.go

@@ -5,6 +5,14 @@ import (
 )
 
 var subtractOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: "(.a - .b) as $x",
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
 	{
 		description:    "Number subtraction - float",
 		subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.",

pkg/yqlib/operator_tag.go

@@ -12,7 +12,7 @@ func assignTagOperator(d *dataTreeNavigator, context Context, expressionNode *Ex
 	tag := ""
 
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -32,7 +32,7 @@ func assignTagOperator(d *dataTreeNavigator, context Context, expressionNode *Ex
 		candidate := el.Value.(*CandidateNode)
 		log.Debugf("Setting tag of : %v", candidate.GetKey())
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}

pkg/yqlib/operator_traverse_path.go

@@ -14,6 +14,7 @@ type traversePreferences struct {
 	IncludeMapKeys       bool
 	DontAutoCreate       bool // by default, we automatically create entries on the fly.
 	DontIncludeMapValues bool
+	OptionalTraverse     bool // e.g. .adf?
 }
 
 func splat(d *dataTreeNavigator, context Context, prefs traversePreferences) (Context, error) {
@@ -60,7 +61,7 @@ func traverse(d *dataTreeNavigator, context Context, matchingNode *CandidateNode
 
 	case yaml.SequenceNode:
 		log.Debug("its a sequence of %v things!", len(value.Content))
-		return traverseArray(matchingNode, operation)
+		return traverseArray(matchingNode, operation, operation.Preferences.(traversePreferences))
 
 	case yaml.AliasNode:
 		log.Debug("its an alias!")
@@ -88,15 +89,20 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
 
 	// rhs is a collect expression that will yield indexes to retreive of the arrays
 
-	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
+
 	if err != nil {
 		return Context{}, err
 	}
+	prefs := traversePreferences{}
 
+	if expressionNode.Rhs.Rhs != nil && expressionNode.Rhs.Rhs.Operation.Preferences != nil {
+		prefs = expressionNode.Rhs.Rhs.Operation.Preferences.(traversePreferences)
+	}
 	var indicesToTraverse = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Content
 
 	//now we traverse the result of the lhs against the indices we found
-	result, err := traverseNodesWithArrayIndices(lhs, indicesToTraverse, traversePreferences{})
+	result, err := traverseNodesWithArrayIndices(lhs, indicesToTraverse, prefs)
 	if err != nil {
 		return Context{}, err
 	}
@@ -130,7 +136,7 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT
 		matchingNode.Node = node.Alias
 		return traverseArrayIndices(context, matchingNode, indicesToTraverse, prefs)
 	} else if node.Kind == yaml.SequenceNode {
-		return traverseArrayWithIndices(matchingNode, indicesToTraverse)
+		return traverseArrayWithIndices(matchingNode, indicesToTraverse, prefs)
 	} else if node.Kind == yaml.MappingNode {
 		return traverseMapWithIndices(context, matchingNode, indicesToTraverse, prefs)
 	} else if node.Kind == yaml.DocumentNode {
@@ -159,7 +165,7 @@ func traverseMapWithIndices(context Context, candidate *CandidateNode, indices [
 	return matchingNodeMap, nil
 }
 
-func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*list.List, error) {
+func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs traversePreferences) (*list.List, error) {
 	log.Debug("traverseArrayWithIndices")
 	var newMatches = list.New()
 	node := unwrapDoc(candidate.Node)
@@ -177,6 +183,9 @@ func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*
 	for _, indexNode := range indices {
 		log.Debug("traverseArrayWithIndices: '%v'", indexNode.Value)
 		index, err := strconv.ParseInt(indexNode.Value, 10, 64)
+		if err != nil && prefs.OptionalTraverse {
+			continue
+		}
 		if err != nil {
 			return nil, fmt.Errorf("Cannot index array with '%v' (%v)", indexNode.Value, err)
 		}
@@ -297,8 +306,8 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *C
 	return nil
 }
 
-func traverseArray(candidate *CandidateNode, operation *Operation) (*list.List, error) {
+func traverseArray(candidate *CandidateNode, operation *Operation, prefs traversePreferences) (*list.List, error) {
 	log.Debug("operation Value %v", operation.Value)
-	indices := []*yaml.Node{&yaml.Node{Value: operation.StringValue}}
+	indices := []*yaml.Node{{Value: operation.StringValue}}
-	return traverseArrayWithIndices(candidate, indices)
+	return traverseArrayWithIndices(candidate, indices, prefs)
 }

pkg/yqlib/operator_traverse_path_test.go

@@ -27,6 +27,22 @@ foobar:
 `
 
 var traversePathOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `[[1]]`,
+		expression: `.[0][0]`,
+		expected: []string{
+			"D0, P[0 0], (!!int)::1\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[[[1]]]`,
+		expression: `.[0][0][0]`,
+		expected: []string{
+			"D0, P[0 0 0], (!!int)::1\n",
+		},
+	},
 	{
 		description: "Simple map navigation",
 		document:    `{a: {b: apple}}`,
@@ -45,6 +61,13 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[1], (!!map)::{c: banana}\n",
 		},
 	},
+	{
+		description:    "Optional Splat",
+		subdescription: "Just like splat, but won't error if you run it against scalars",
+		document:       `"cat"`,
+		expression:     `.[]`,
+		expected:       []string{},
+	},
 	{
 		description:    "Special characters",
 		subdescription: "Use quotes with brackets around path elements with special characters",
@@ -71,6 +94,14 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[flying fox], (!!str)::frog\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `c: dog`,
+		expression: `.[.a.b] as $x`,
+		expected: []string{
+			"D0, P[], (doc)::c: dog\n",
+		},
+	},
 	{
 		description:    "Dynamic keys",
 		subdescription: `Expressions within [] can be used to dynamically lookup / calculate keys`,
@@ -97,6 +128,19 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[a b], (!!null)::null\n",
 		},
 	},
+	{
+		description:    "Optional identifier",
+		subdescription: "Like jq, does not output an error when the yaml is not an array or object as expected",
+		document:       `[1,2,3]`,
+		expression:     `.a?`,
+		expected:       []string{},
+	},
+	{
+		skipDoc:    true,
+		document:   `[[1,2,3], {a: frog}]`,
+		expression: `.[] | .["a"]?`,
+		expected:   []string{"D0, P[1 a], (!!str)::frog\n"},
+	},
 	{
 		skipDoc:    true,
 		document:   ``,

pkg/yqlib/operator_union_test.go

@@ -5,6 +5,20 @@ import (
 )
 
 var unionOperatorScenarios = []expressionScenario{
+	// {
+	// 	skipDoc:    true,
+	// 	document:   "{}",
+	// 	expression: `(.a, .b.c) as $x`,
+	// 	expected: []string{
+	// 		"D0, P[], (doc)::{}\n",
+	// 	},
+	// },
+	// {
+	// 	skipDoc:    true,
+	// 	document:   "{}",
+	// 	expression: `(.a, .b.c)`,
+	// 	expected:   []string{},
+	// },
 	{
 		description: "Combine scalars",
 		expression:  `1, true, "cat"`,

pkg/yqlib/operator_unique.go

@@ -0,0 +1,64 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+
+	"github.com/elliotchance/orderedmap"
+	yaml "gopkg.in/yaml.v3"
+)
+
+func unique(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
+	uniqueByExpression := &ExpressionNode{Operation: &Operation{OperationType: uniqueByOpType}, Rhs: selfExpression}
+	return uniqueBy(d, context, uniqueByExpression)
+
+}
+
+func uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	log.Debugf("-- uniqueBy Operator")
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+
+		if candidateNode.Kind != yaml.SequenceNode {
+			return Context{}, fmt.Errorf("Only arrays are supported for unique")
+		}
+
+		var newMatches = orderedmap.NewOrderedMap()
+		for _, node := range candidateNode.Content {
+			child := &CandidateNode{Node: node}
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(child), expressionNode.Rhs)
+
+			if err != nil {
+				return Context{}, err
+			}
+
+			keyValue := "null"
+
+			if rhs.MatchingNodes.Len() > 0 {
+				first := rhs.MatchingNodes.Front()
+				keyCandidate := first.Value.(*CandidateNode)
+				keyValue = keyCandidate.Node.Value
+			}
+
+			_, exists := newMatches.Get(keyValue)
+
+			if !exists {
+				newMatches.Set(keyValue, child.Node)
+			}
+		}
+		resultNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+		for el := newMatches.Front(); el != nil; el = el.Next() {
+			resultNode.Content = append(resultNode.Content, el.Value.(*yaml.Node))
+		}
+
+		results.PushBack(candidate.CreateChild(nil, resultNode))
+	}
+
+	return context.ChildContext(results), nil
+
+}

pkg/yqlib/operator_unique_test.go

@@ -0,0 +1,65 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var uniqueOperatorScenarios = []expressionScenario{
+	{
+		description: "Unique array of scalars (string/numbers)",
+		document:    `[1,2,3,2]`,
+		expression:  `unique`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 1\n- 2\n- 3\n",
+		},
+	},
+	{
+		description:    "Unique nulls",
+		subdescription: "Unique works on the node value, so it considers different representations of nulls to be different",
+		document:       `[~,null, ~, null]`,
+		expression:     `unique`,
+		expected: []string{
+			"D0, P[], (!!seq)::- ~\n- null\n",
+		},
+	},
+	{
+		description:    "Unique all nulls",
+		subdescription: "Run against the node tag to unique all the nulls",
+		document:       `[~,null, ~, null]`,
+		expression:     `unique_by(tag)`,
+		expected: []string{
+			"D0, P[], (!!seq)::- ~\n",
+		},
+	},
+	{
+		description: "Unique array object fields",
+		document:    `[{name: harry, pet: cat}, {name: billy, pet: dog}, {name: harry, pet: dog}]`,
+		expression:  `unique_by(.name)`,
+		expected: []string{
+			"D0, P[], (!!seq)::- {name: harry, pet: cat}\n- {name: billy, pet: dog}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`,
+		expression: `unique_by(.name)`,
+		expected: []string{
+			"D0, P[], (!!seq)::- {name: harry, pet: cat}\n- {pet: fish}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`,
+		expression: `unique_by(.cat.dog)`,
+		expected: []string{
+			"D0, P[], (!!seq)::- {name: harry, pet: cat}\n",
+		},
+	},
+}
+
+func TestUniqueOperatorScenarios(t *testing.T) {
+	for _, tt := range uniqueOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Unique", uniqueOperatorScenarios)
+}

pkg/yqlib/operator_variables.go

@@ -16,7 +16,7 @@ func getVariableOperator(d *dataTreeNavigator, context Context, expressionNode *
 }
 
 func assignVariableOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
+	lhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Lhs)
 	if err != nil {
 		return Context{}, nil
 	}

pkg/yqlib/operator_variables_test.go

@@ -5,6 +5,14 @@ import (
 )
 
 var variableOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: `.a.b as $foo`,
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
 	{
 		description: "Single value variable",
 		document:    `a: cat`,

pkg/yqlib/operators.go

@@ -24,7 +24,19 @@ func emptyOperator(d *dataTreeNavigator, context Context, expressionNode *Expres
 
 type crossFunctionCalculation func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)
 
-func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *CandidateNode, rhs Context, calculation crossFunctionCalculation, results *list.List) error {
+func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *CandidateNode, rhs Context, calculation crossFunctionCalculation, results *list.List, calcWhenEmpty bool) error {
+
+	if calcWhenEmpty && rhs.MatchingNodes.Len() == 0 {
+		resultCandidate, err := calculation(d, context, lhsCandidate, nil)
+		if err != nil {
+			return err
+		}
+		if resultCandidate != nil {
+			results.PushBack(resultCandidate)
+		}
+		return nil
+	}
+
 	for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() {
 		log.Debugf("Applying calc")
 		rhsCandidate := rightEl.Value.(*CandidateNode)
@@ -32,7 +44,9 @@ func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *Candidat
 		if err != nil {
 			return err
 		}
-		results.PushBack(resultCandidate)
+		if resultCandidate != nil {
+			results.PushBack(resultCandidate)
+		}
 	}
 	return nil
 }
@@ -52,14 +66,7 @@ func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	}
 
 	if calcWhenEmpty && lhs.MatchingNodes.Len() == 0 {
-		if rhs.MatchingNodes.Len() == 0 {
+		err := resultsForRhs(d, context, nil, rhs, calculation, results, calcWhenEmpty)
-			resultCandidate, err := calculation(d, context, nil, nil)
-			if err != nil {
-				return Context{}, err
-			}
-			results.PushBack(resultCandidate)
-		}
-		err := resultsForRhs(d, context, nil, rhs, calculation, results)
 		if err != nil {
 			return Context{}, err
 		}
@@ -68,7 +75,7 @@ func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
 		lhsCandidate := el.Value.(*CandidateNode)
 
-		err := resultsForRhs(d, context, lhsCandidate, rhs, calculation, results)
+		err := resultsForRhs(d, context, lhsCandidate, rhs, calculation, results, calcWhenEmpty)
 		if err != nil {
 			return Context{}, err
 		}

pkg/yqlib/operators_test.go

@@ -71,7 +71,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 		t.Error(fmt.Errorf("%v: %v", err, s.expression))
 		return
 	}
-	test.AssertResultComplexWithContext(t, s.expected, resultsToString(context.MatchingNodes), fmt.Sprintf("exp: %v\ndoc: %v", s.expression, s.document))
+	test.AssertResultComplexWithContext(t, s.expected, resultsToString(context.MatchingNodes), fmt.Sprintf("desc: %v\nexp: %v\ndoc: %v", s.description, s.expression, s.document))
 }
 
 func resultsToString(results *list.List) []string {

scripts/format.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
 find . \( -path ./vendor \) -prune -o -name "*.go" -exec goimports -w {} \;
+gofmt -w -s .
 go mod tidy
 go mod vendor

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.7.1'
+version: '4.9.3'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.