Increment version

cmd/evaluate_all_command.go

@@ -17,6 +17,9 @@ func createEvaluateAllCommand() *cobra.Command {
 		Example: `
 # merges f2.yml into f1.yml (inplace)
 yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml		
+
+# use '-' as a filename to read from STDIN
+cat file2.yml | yq ea '.a.b' file1.yml - file3.yml
 `,
 		Long: "Evaluate All:\nUseful when you need to run an expression across several yaml documents or files. Consumes more memory than eval",
 		RunE: evaluateAll,

cmd/evalute_sequence_command.go

@@ -21,6 +21,9 @@ yq e '.a.b | length' f1.yml f2.yml
 # prints out the file
 yq e sample.yaml 
 
+# use '-' as a filename to read from STDIN
+cat file2.yml | yq e '.a.b' file1.yml - file3.yml
+
 # prints a new yaml document
 yq e -n '.a.b.c = "cat"' 
 

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.6.3"
+	Version = "4.8.0"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.6.3
+FROM mikefarah/yq:4.8.0
 
 COPY entrypoint.sh /entrypoint.sh
 

pkg/yqlib/candidate_node.go

@@ -8,9 +8,10 @@ import (
 )
 
 type CandidateNode struct {
-	Node      *yaml.Node    // the actual node
-	Path      []interface{} /// the path we took to get to this node
-	Document  uint          // the document index of this node
+	Node      *yaml.Node     // the actual node
+	Parent    *CandidateNode // parent node
+	Path      []interface{}  /// the path we took to get to this node
+	Document  uint           // the document index of this node
 	Filename  string
 	FileIndex int
 	// when performing op against all nodes given, this will treat all the nodes as one
@@ -31,6 +32,7 @@ func (n *CandidateNode) CreateChild(path interface{}, node *yaml.Node) *Candidat
 	return &CandidateNode{
 		Node:      node,
 		Path:      n.createChildPath(path),
+		Parent:    n,
 		Document:  n.Document,
 		Filename:  n.Filename,
 		FileIndex: n.FileIndex,

pkg/yqlib/doc/Anchor and Alias Operators.md

@@ -1,8 +1,99 @@
-Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference aliases and remove anchor names).
+Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
 
 `yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
 
 
+## Merge one map
+see https://yaml.org/type/merge.html
+
+Given a sample.yml file of:
+```yaml
+- &CENTER
+  x: 1
+  y: 2
+- &LEFT
+  x: 0
+  y: 2
+- &BIG
+  r: 10
+- &SMALL
+  r: 1
+- !!merge <<: *CENTER
+  r: 10
+```
+then
+```bash
+yq eval '.[4] | explode(.)' sample.yml
+```
+will output
+```yaml
+x: 1
+y: 2
+r: 10
+```
+
+## Merge multiple maps
+see https://yaml.org/type/merge.html
+
+Given a sample.yml file of:
+```yaml
+- &CENTER
+  x: 1
+  y: 2
+- &LEFT
+  x: 0
+  y: 2
+- &BIG
+  r: 10
+- &SMALL
+  r: 1
+- !!merge <<:
+    - *CENTER
+    - *BIG
+```
+then
+```bash
+yq eval '.[4] | explode(.)' sample.yml
+```
+will output
+```yaml
+r: 10
+x: 1
+y: 2
+```
+
+## Override
+see https://yaml.org/type/merge.html
+
+Given a sample.yml file of:
+```yaml
+- &CENTER
+  x: 1
+  y: 2
+- &LEFT
+  x: 0
+  y: 2
+- &BIG
+  r: 10
+- &SMALL
+  r: 1
+- !!merge <<:
+    - *BIG
+    - *LEFT
+    - *SMALL
+  x: 1
+```
+then
+```bash
+yq eval '.[4] | explode(.)' sample.yml
+```
+will output
+```yaml
+r: 10
+x: 1
+y: 2
+```
+
 ## Get anchor
 Given a sample.yml file of:
 ```yaml
@@ -183,9 +274,9 @@ bar:
   c: bar_c
 foobarList:
   b: bar_b
-  a: foo_a
-  thing: bar_thing
+  thing: foo_thing
   c: foobarList_c
+  a: foo_a
 foobar:
   c: foo_c
   a: foo_a

pkg/yqlib/doc/Comment Operators.md

@@ -88,7 +88,7 @@ a: cat
 b: dog # leave this
 ```
 
-## Remove all comments
+## Remove (strip) all comments
 Note the use of `...` to ensure key nodes are included.
 
 Given a sample.yml file of:

pkg/yqlib/doc/Entries.md

@@ -0,0 +1,87 @@
+Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps.
+## to_entries Map
+Given a sample.yml file of:
+```yaml
+a: 1
+b: 2
+```
+then
+```bash
+yq eval 'to_entries' sample.yml
+```
+will output
+```yaml
+- key: a
+  value: 1
+- key: b
+  value: 2
+```
+
+## to_entries Array
+Given a sample.yml file of:
+```yaml
+- a
+- b
+```
+then
+```bash
+yq eval 'to_entries' sample.yml
+```
+will output
+```yaml
+- key: 0
+  value: a
+- key: 1
+  value: b
+```
+
+## from_entries map
+Given a sample.yml file of:
+```yaml
+a: 1
+b: 2
+```
+then
+```bash
+yq eval 'to_entries | from_entries' sample.yml
+```
+will output
+```yaml
+a: 1
+b: 2
+```
+
+## from_entries with numeric key indexes
+from_entries always creates a map, even for numeric keys
+
+Given a sample.yml file of:
+```yaml
+- a
+- b
+```
+then
+```bash
+yq eval 'to_entries | from_entries' sample.yml
+```
+will output
+```yaml
+0: a
+1: b
+```
+
+## Use with_entries to update keys
+Given a sample.yml file of:
+```yaml
+a: 1
+b: 2
+```
+then
+```bash
+yq eval 'with_entries(.key |= "KEY_" + .)' sample.yml
+```
+will output
+```yaml
+KEY_a: 1
+KEY_b: 2
+```
+

pkg/yqlib/doc/Equals.md

@@ -93,3 +93,31 @@ will output
 true
 ```
 
+## Non exisitant key doesn't equal a value
+Given a sample.yml file of:
+```yaml
+a: frog
+```
+then
+```bash
+yq eval 'select(.b != "thing")' sample.yml
+```
+will output
+```yaml
+a: frog
+```
+
+## Two non existant keys are equal
+Given a sample.yml file of:
+```yaml
+a: frog
+```
+then
+```bash
+yq eval 'select(.b == .c)' sample.yml
+```
+will output
+```yaml
+a: frog
+```
+

pkg/yqlib/doc/Multiply (Merge).md

@@ -231,6 +231,46 @@ will output
   age: 32
 ```
 
+## Merge arrays of objects together, matching on a key
+It's a complex command, the trickyness comes from needing to have the right context in the expressions.
+First we save the second array into a variable '$two' which lets us reference it later.
+We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. 
+We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter.
+
+
+Given a sample.yml file of:
+```yaml
+- a: apple
+  b: appleB
+- a: kiwi
+  b: kiwiB
+- a: banana
+  b: bananaB
+```
+And another sample another.yml file of:
+```yaml
+- a: banana
+  c: bananaC
+- a: apple
+  b: appleB2
+- a: dingo
+  c: dingoC
+```
+then
+```bash
+yq eval-all '(select(fi==1) | .[]) as $two | select(fi==0) | .[] |= (. as $cur |  $cur * ($two | select(.a == $cur.a)))' sample.yml another.yml
+```
+will output
+```yaml
+- a: apple
+  b: appleB2
+- a: kiwi
+  b: kiwiB
+- a: banana
+  b: bananaB
+  c: bananaC
+```
+
 ## Merge to prefix an element
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/String Operators.md

@@ -18,6 +18,42 @@ will output
 cat; meow; 1; ; true
 ```
 
+## Substitute / Replace string
+This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
+Note the use of `|=` to run in context of the current string value.
+
+Given a sample.yml file of:
+```yaml
+a: dogs are great
+```
+then
+```bash
+yq eval '.a |= sub("dogs", "cats")' sample.yml
+```
+will output
+```yaml
+a: cats are great
+```
+
+## Substitute / Replace string with regex
+This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
+Note the use of `|=` to run in context of the current string value.
+
+Given a sample.yml file of:
+```yaml
+a: cat
+b: heat
+```
+then
+```bash
+yq eval '.[] |= sub("(a)", "${1}r")' sample.yml
+```
+will output
+```yaml
+a: cart
+b: heart
+```
+
 ## Split strings
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Style.md

@@ -1,4 +1,42 @@
 The style operator can be used to get or set the style of nodes (e.g. string style, yaml style)
+## Update and set style of a particular node (simple)
+Given a sample.yml file of:
+```yaml
+a:
+  b: thing
+  c: something
+```
+then
+```bash
+yq eval '.a.b = "new" | .a.b style="double"' sample.yml
+```
+will output
+```yaml
+a:
+  b: "new"
+  c: something
+```
+
+## Update and set style of a particular node using path variables
+You can use a variable to re-use a path
+
+Given a sample.yml file of:
+```yaml
+a:
+  b: thing
+  c: something
+```
+then
+```bash
+yq eval '.a.b as $x | $x = "new" | $x style="double"' sample.yml
+```
+will output
+```yaml
+a:
+  b: "new"
+  c: something
+```
+
 ## Set tagged style
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Traverse (Read).md

@@ -32,6 +32,21 @@ b: apple
 c: banana
 ```
 
+## Optional Splat
+Just like splat, but won't error if you run it against scalars
+
+Given a sample.yml file of:
+```yaml
+cat
+```
+then
+```bash
+yq eval '.[]' sample.yml
+```
+will output
+```yaml
+```
+
 ## Special characters
 Use quotes with brackets around path elements with special characters
 
@@ -98,6 +113,23 @@ will output
 null
 ```
 
+## Optional identifier
+Like jq, does not output an error when the yaml is not an array or object as expected
+
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+```
+then
+```bash
+yq eval '.a?' sample.yml
+```
+will output
+```yaml
+```
+
 ## Wildcard matching
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/aa.md

@@ -1,12 +1,81 @@
 # Operators
 
-In `yq` expressions are made up of operators. Operators have 0-2 arguments and run against the current 'matching' nodes in the expression tree.
+In `yq` expressions are made up of operators and pipes. A context of nodes is passed through the expression and each operation takes the context as input and returns a new context as output. That output is piped in as input for the next operation in the expression. To begin with, the context is set to the first yaml document of the first yaml file (if processing in sequence using eval).
 
 Lets look at a couple of examples.
 
-The `length` operator take no arguments, and will simply return the length of _each_ matching node. So if there were 2 nodes, one string and one array, length will update the 'matching' nodes context to be two new numeric scalar nodes representing the lengths of the orignal 'matching' nodes.
+## Example with a simple operator
 
-The `=` operator takes two arguments, a `lhs` expression and `rhs` expression. It runs the 'matching' nodes context against the `lhs` expression to find the nodes to update, lets call it `lhsNodes`, and then runs the matching nodes against the `rhs` to find the new values, lets call that `rhsNodes`. It updates the `lhsNodes` values with the `rhsNodes` values and _returns the original matching nodes_. This is important, where length changed the matching nodes to be new nodes with the length values, `=` returns the original matching nodes, albeit with some of the nodes values updated. So `.a = 3` will still return the parent matching node, but with the matching child updated.
+Given a document like:
 
-Please see the individual operator docs for more information and examples.
+```yaml
+- [a]
+- "cat"
+```
 
+with an expression:
+
+```
+.[] | length
+```
+
+`yq` will initially set the context as single node of the entire yaml document, an array of two elements.
+
+```yaml
+- [a]
+- "cat"
+```
+
+This gets piped into the splat operator `.[]` which will split out the context into a collection of two nodes `[a]` and `"cat"`. Note that this is _not_ a yaml array.
+
+The `length` operator take no arguments, and will simply return the length of _each_ matching node in the context. So for the context of `[a]` and `"cat"`, it will return a new context of `1` and `3`.
+
+This being the last operation in the expression, the results will be printed out:
+
+```
+1
+3
+```
+
+# Example with an operator that takes arguments.
+
+Given a document like:
+
+```yaml
+a: cat
+b: dog
+```
+
+with an expression:
+
+```
+.a = .b
+```
+
+The `=` operator takes two arguments, a `lhs` expression, which in this case is `.a` and `rhs` expression which is `.b`. 
+
+It pipes the current, lets call it 'root' context through the `lhs` expression of `.a` to return the node 
+
+```yaml
+cat
+```
+
+Note that this node holds not only its value 'cat', but comments and metadata too, including path and parent information.
+
+The `=` operator then pipes the 'root' context through the `rhs` expression of `.b` to retun the node
+
+```yaml
+dog
+```
+
+Both sides have now been evaluated, so the operator performs its actual operation of assignment, and copies across the value from the RHS to the value on the LHS, and it returns the now updated 'root' context:
+
+```yaml
+a: cat
+b: dog
+```
+
+# Relative update (e.g. `|=`)
+There is another form of the `=` operator which we call the relative form. It's very similar to `=` but with one key differnce when evaluating the RHS expression.
+
+In the plain form, we pass in the 'root' level context to the RHS expresssion. In relative form, we pass in each result of the LHS to the RHS expression. Let's go through an example.

pkg/yqlib/doc/headers/Anchor and Alias Operators.md

@@ -1,4 +1,4 @@
-Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference aliases and remove anchor names).
+Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
 
 `yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
 

pkg/yqlib/doc/headers/Entries.md

@@ -0,0 +1 @@
+Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps.

pkg/yqlib/expression_postfix.go

@@ -55,7 +55,15 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			log.Debugf("deleteing open bracket from opstack")
 
 			//and append a collect to the opStack
-			result = append(result, &Operation{OperationType: collectOperator})
+			// hack - see if there's the optional traverse flag
+			// on the close op - move it to the collect op.
+			// allows for .["cat"]?
+			prefs := traversePreferences{}
+			closeTokenMatch := string(currentToken.Match.Bytes)
+			if closeTokenMatch[len(closeTokenMatch)-1:] == "?" {
+				prefs.OptionalTraverse = true
+			}
+			result = append(result, &Operation{OperationType: collectOperator, Preferences: prefs})
 			log.Debugf("put collect onto the result")
 			result = append(result, &Operation{OperationType: shortPipeOpType})
 			log.Debugf("put shortpipe onto the result")

pkg/yqlib/expression_processing_test.go

@@ -62,6 +62,11 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"),
 		append(make([]interface{}, 0), "b", "a", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
 	},
+	{
+		`.b[.a]?`,
+		append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"),
+		append(make([]interface{}, 0), "b", "a", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
 	{
 		`.[]`,
 		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
@@ -72,6 +77,11 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
 		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
 	},
+	{
+		`.a[]?`,
+		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
+		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
 	{
 		`.a.[]`,
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
@@ -82,6 +92,11 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
 		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
 	},
+	{
+		`.a[0]?`,
+		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
+		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
 	{
 		`.a.[0]`,
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),

pkg/yqlib/expression_tokeniser.go

@@ -29,8 +29,9 @@ const (
 type token struct {
 	TokenType            tokenType
 	Operation            *Operation
-	AssignOperation      *Operation // e.g. tag (GetTag) op becomes AssignTag if '=' follows it
-	CheckForPostTraverse bool       // e.g. [1]cat should really be [1].cat
+	AssignOperation      *Operation      // e.g. tag (GetTag) op becomes AssignTag if '=' follows it
+	CheckForPostTraverse bool            // e.g. [1]cat should really be [1].cat
+	Match                *machines.Match // match that created this token
 
 }
 
@@ -63,12 +64,19 @@ func (t *token) toString(detail bool) string {
 func pathToken(wrapped bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		value := string(m.Bytes)
+		prefs := traversePreferences{}
+
+		if value[len(value)-1:] == "?" {
+			prefs.OptionalTraverse = true
+			value = value[:len(value)-1]
+		}
+
 		value = value[1:]
 		if wrapped {
 			value = unwrap(value)
 		}
 		log.Debug("PathToken %v", value)
-		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: traversePreferences{}}
+		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs}
 		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
 	}
 }
@@ -138,7 +146,7 @@ func assignAllCommentsOp(updateAssign bool) lex.Action {
 
 func literalToken(pType tokenType, checkForPost bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &token{TokenType: pType, CheckForPostTraverse: checkForPost}, nil
+		return &token{TokenType: pType, CheckForPostTraverse: checkForPost, Match: m}, nil
 	}
 }
 
@@ -264,6 +272,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`splitDoc`), opToken(splitDocumentOpType))
 
 	lexer.Add([]byte(`join`), opToken(joinStringOpType))
+	lexer.Add([]byte(`sub`), opToken(subStringOpType))
+
 	lexer.Add([]byte(`split`), opToken(splitStringOpType))
 	lexer.Add([]byte(`keys`), opToken(keysOpType))
 
@@ -276,6 +286,9 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`fileIndex`), opToken(getFileIndexOpType))
 	lexer.Add([]byte(`fi`), opToken(getFileIndexOpType))
 	lexer.Add([]byte(`path`), opToken(getPathOpType))
+	lexer.Add([]byte(`to_entries`), opToken(toEntriesOpType))
+	lexer.Add([]byte(`from_entries`), opToken(fromEntriesOpType))
+	lexer.Add([]byte(`with_entries`), opToken(withEntriesOpType))
 
 	lexer.Add([]byte(`lineComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
 
@@ -298,8 +311,8 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
 
-	lexer.Add([]byte(`\."[^ "]+"`), pathToken(true))
-	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+`), pathToken(false))
+	lexer.Add([]byte(`\."[^ "]+"\??`), pathToken(true))
+	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+\??`), pathToken(false))
 	lexer.Add([]byte(`\.`), selfToken())
 
 	lexer.Add([]byte(`\|`), opToken(pipeOpType))
@@ -319,7 +332,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`env\([^\)]+\)`), envOp(false))
 
 	lexer.Add([]byte(`\[`), literalToken(openCollect, false))
-	lexer.Add([]byte(`\]`), literalToken(closeCollect, true))
+	lexer.Add([]byte(`\]\??`), literalToken(closeCollect, true))
 	lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
 	lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
 	lexer.Add([]byte(`\*[\+|\?d]*`), multiplyWithPrefs())

pkg/yqlib/lib.go

@@ -60,6 +60,11 @@ var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence:
 
 var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: collectOperator}
+
+var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: 50, Handler: toEntriesOperator}
+var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator}
+var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator}
+
 var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator}
 var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator}
 var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}
@@ -75,6 +80,7 @@ var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50,
 var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator}
 var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
 var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}
+var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator}
 var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
 
 var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}
@@ -94,7 +100,6 @@ var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs:
 var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator}
 var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator}
 var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
-var deleteImmediateChildOpType = &operationType{Type: "DELETE_IMMEDIATE_CHILD", NumArgs: 1, Precedence: 40, Handler: deleteImmediateChildOperator}
 
 type Operation struct {
 	OperationType *operationType

pkg/yqlib/operator_add.go

@@ -39,7 +39,7 @@ func toNodes(candidate *CandidateNode) []*yaml.Node {
 func addOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("Add operator")
 
-	return crossFunction(d, context, expressionNode, add)
+	return crossFunction(d, context, expressionNode, add, false)
 }
 
 func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

pkg/yqlib/operator_alternative.go

@@ -1,14 +1,14 @@
 package yqlib
 
-// corssFunction no matches
-// can boolean use crossfunction
-
 func alternativeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- alternative")
-	return crossFunction(d, context, expressionNode, alternativeFunc)
+	return crossFunction(d, context, expressionNode, alternativeFunc, true)
 }
 
 func alternativeFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	if lhs == nil {
+		return rhs, nil
+	}
 	lhs.Node = unwrapDoc(lhs.Node)
 	rhs.Node = unwrapDoc(rhs.Node)
 	log.Debugf("Alternative LHS: %v", lhs.Node.Tag)

pkg/yqlib/operator_alternative_test.go

@@ -13,6 +13,14 @@ var alternativeOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!str)::bridge\n",
 		},
 	},
+	{
+		expression: `select(tag == "seq") // "cat"`,
+		skipDoc:    true,
+		document:   `a: frog`,
+		expected: []string{
+			"D0, P[], (!!str)::cat\n",
+		},
+	},
 	{
 		description: "LHS is not defined",
 		expression:  `.a // "hello"`,

pkg/yqlib/operator_anchors_aliases.go

@@ -176,7 +176,7 @@ func explodeNode(node *yaml.Node, context Context) error {
 			} else {
 				if valueNode.Kind == yaml.SequenceNode {
 					log.Debugf("an alias merge list!")
-					for index := 0; index < len(valueNode.Content); index = index + 1 {
+					for index := len(valueNode.Content) - 1; index >= 0; index = index - 1 {
 						aliasNode := valueNode.Content[index]
 						err := applyAlias(node, aliasNode.Alias, index, context.ChildContext(newContent))
 						if err != nil {

pkg/yqlib/operator_anchors_aliases_test.go

@@ -4,7 +4,36 @@ import (
 	"testing"
 )
 
+var specDocument = `- &CENTER { x: 1, y: 2 }
+- &LEFT { x: 0, y: 2 }
+- &BIG { r: 10 }
+- &SMALL { r: 1 }
+`
+
+var expectedSpecResult = "D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n"
+
 var anchorOperatorScenarios = []expressionScenario{
+	{
+		description:    "Merge one map",
+		subdescription: "see https://yaml.org/type/merge.html",
+		document:       specDocument + "- << : *CENTER\n  r: 10\n",
+		expression:     ".[4] | explode(.)",
+		expected:       []string{expectedSpecResult},
+	},
+	{
+		description:    "Merge multiple maps",
+		subdescription: "see https://yaml.org/type/merge.html",
+		document:       specDocument + "- << : [ *CENTER, *BIG ]\n",
+		expression:     ".[4] | explode(.)",
+		expected:       []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"},
+	},
+	{
+		description:    "Override",
+		subdescription: "see https://yaml.org/type/merge.html",
+		document:       specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n  x: 1\n",
+		expression:     ".[4] | explode(.)",
+		expected:       []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"},
+	},
 	{
 		description: "Get anchor",
 		document:    `a: &billyBob cat`,
@@ -91,9 +120,9 @@ bar:
     c: bar_c
 foobarList:
     b: bar_b
-    a: foo_a
-    thing: bar_thing
+    thing: foo_thing
     c: foobarList_c
+    a: foo_a
 foobar:
     c: foo_c
     a: foo_a
@@ -106,7 +135,7 @@ foobar:
 		expression: `.foo* | explode(.) | (. style="flow")`,
 		expected: []string{
 			"D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n",
-			"D0, P[foobarList], (!!map)::{b: bar_b, a: foo_a, thing: bar_thing, c: foobarList_c}\n",
+			"D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n",
 			"D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n",
 		},
 	},
@@ -116,7 +145,7 @@ foobar:
 		expression: `.foo* | explode(explode(.)) | (. style="flow")`,
 		expected: []string{
 			"D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n",
-			"D0, P[foobarList], (!!map)::{b: bar_b, a: foo_a, thing: bar_thing, c: foobarList_c}\n",
+			"D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n",
 			"D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n",
 		},
 	},

pkg/yqlib/operator_booleans.go

@@ -27,20 +27,37 @@ type boolOp func(bool, bool) bool
 
 func performBoolOp(op boolOp) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-		lhs.Node = unwrapDoc(lhs.Node)
-		rhs.Node = unwrapDoc(rhs.Node)
+		owner := lhs
 
-		lhsTrue, errDecoding := isTruthy(lhs)
-		if errDecoding != nil {
-			return nil, errDecoding
+		if lhs == nil && rhs == nil {
+			owner = &CandidateNode{}
+		} else if lhs == nil {
+			owner = rhs
 		}
 
-		rhsTrue, errDecoding := isTruthy(rhs)
-		if errDecoding != nil {
-			return nil, errDecoding
-		}
+		var errDecoding error
+		lhsTrue := false
+		if lhs != nil {
+			lhs.Node = unwrapDoc(lhs.Node)
+			lhsTrue, errDecoding = isTruthy(lhs)
 
-		return createBooleanCandidate(lhs, op(lhsTrue, rhsTrue)), nil
+			if errDecoding != nil {
+				return nil, errDecoding
+			}
+		}
+		log.Debugf("-- lhsTrue", lhsTrue)
+
+		rhsTrue := false
+		if rhs != nil {
+			rhs.Node = unwrapDoc(rhs.Node)
+			rhsTrue, errDecoding = isTruthy(rhs)
+			if errDecoding != nil {
+				return nil, errDecoding
+			}
+		}
+		log.Debugf("-- rhsTrue", rhsTrue)
+
+		return createBooleanCandidate(owner, op(lhsTrue, rhsTrue)), nil
 	}
 }
 
@@ -48,8 +65,9 @@ func orOperator(d *dataTreeNavigator, context Context, expressionNode *Expressio
 	log.Debugf("-- orOp")
 	return crossFunction(d, context, expressionNode, performBoolOp(
 		func(b1 bool, b2 bool) bool {
+			log.Debugf("-- peformingOrOp with %v and %v", b1, b2)
 			return b1 || b2
-		}))
+		}), true)
 }
 
 func andOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@@ -57,7 +75,7 @@ func andOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	return crossFunction(d, context, expressionNode, performBoolOp(
 		func(b1 bool, b2 bool) bool {
 			return b1 && b2
-		}))
+		}), true)
 }
 
 func notOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {

pkg/yqlib/operator_booleans_test.go

@@ -12,6 +12,22 @@ var booleanOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::true\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "b: hi",
+		expression: `select(.a or .b)`,
+		expected: []string{
+			"D0, P[], (doc)::b: hi\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "b: hi",
+		expression: `select((.a and .b) | not)`,
+		expected: []string{
+			"D0, P[], (doc)::b: hi\n",
+		},
+	},
 	{
 		description: "AND example",
 		expression:  `true and false`,

pkg/yqlib/operator_comments_test.go

@@ -71,7 +71,7 @@ var commentOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description:    "Remove all comments",
+		description:    "Remove (strip) all comments",
 		subdescription: "Note the use of `...` to ensure key nodes are included.",
 		document:       "# hi\n\na: cat # comment\n\n# great\n\nb: # key comment",
 		expression:     `... comments=""`,

pkg/yqlib/operator_create_map.go

@@ -62,7 +62,7 @@ func sequenceFor(d *dataTreeNavigator, context Context, matchingNode *CandidateN
 			}
 
 			return &CandidateNode{Node: &node, Document: document, Path: path}, nil
-		})
+		}, false)
 
 	if err != nil {
 		return nil, err

pkg/yqlib/operator_delete.go

@@ -18,48 +18,23 @@ func deleteChildOperator(d *dataTreeNavigator, context Context, expressionNode *
 	for el := nodesToDelete.MatchingNodes.Back(); el != nil; el = el.Prev() {
 		candidate := el.Value.(*CandidateNode)
 
-		if len(candidate.Path) > 0 {
-			deleteImmediateChildOp := &Operation{
-				OperationType: deleteImmediateChildOpType,
-				Value:         candidate.Path[len(candidate.Path)-1],
-			}
-
-			deleteImmediateChildOpNode := &ExpressionNode{
-				Operation: deleteImmediateChildOp,
-				Rhs:       createTraversalTree(candidate.Path[0:len(candidate.Path)-1], traversePreferences{}, false),
-			}
-
-			_, err := d.GetMatchingNodes(contextToUse, deleteImmediateChildOpNode)
-			if err != nil {
-				return Context{}, err
-			}
+		//problem: context may already be '.a' and then I pass in '.a.a2'.
+		// should pass in .a2.
+		if candidate.Parent == nil {
+			log.Info("Could not find parent of %v", candidate.GetKey())
+			return context, nil
 		}
-	}
-	return context, nil
-}
 
-func deleteImmediateChildOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	parents, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		parentNode := candidate.Parent.Node
+		childPath := candidate.Path[len(candidate.Path)-1]
 
-	if err != nil {
-		return Context{}, err
-	}
-
-	childPath := expressionNode.Operation.Value
-
-	log.Debug("childPath to remove %v", childPath)
-
-	for el := parents.MatchingNodes.Front(); el != nil; el = el.Next() {
-		parent := el.Value.(*CandidateNode)
-		parentNode := unwrapDoc(parent.Node)
 		if parentNode.Kind == yaml.MappingNode {
-			deleteFromMap(parent, childPath)
+			deleteFromMap(candidate.Parent, childPath)
 		} else if parentNode.Kind == yaml.SequenceNode {
-			deleteFromArray(parent, childPath)
+			deleteFromArray(candidate.Parent, childPath)
 		} else {
 			return Context{}, fmt.Errorf("Cannot delete nodes from parent of tag %v", parentNode.Tag)
 		}
-
 	}
 	return context, nil
 }

pkg/yqlib/operator_delete_test.go

@@ -21,6 +21,70 @@ var deleteOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: {a2: frood}}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{a: {a1: fred, a2: frood}}`,
+		expression: `.a | del(.a1)`,
+		expected: []string{
+			"D0, P[a], (!!map)::{a2: frood}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: [1,2,3]`,
+		expression: `.a | del(.[1])`,
+		expected: []string{
+			"D0, P[a], (!!seq)::[1, 3]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[0, {a: cat, b: dog}]`,
+		expression: `.[1] | del(.a)`,
+		expected: []string{
+			"D0, P[1], (!!map)::{b: dog}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{a: cat, b: dog}]`,
+		expression: `.[0] | del(.a)`,
+		expected: []string{
+			"D0, P[0], (!!map)::{b: dog}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{a: {b: thing, c: frog}}]`,
+		expression: `.[0].a | del(.b)`,
+		expected: []string{
+			"D0, P[0 a], (!!map)::{c: frog}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{a: {b: thing, c: frog}}]`,
+		expression: `.[0] | del(.a.b)`,
+		expected: []string{
+			"D0, P[0], (!!map)::{a: {c: frog}}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [0, {b: thing, c: frog}]}`,
+		expression: `.a[1] | del(.b)`,
+		expected: []string{
+			"D0, P[a 1], (!!map)::{c: frog}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [0, {b: thing, c: frog}]}`,
+		expression: `.a | del(.[1].b)`,
+		expected: []string{
+			"D0, P[a], (!!seq)::[0, {c: frog}]\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `{a: {a1: fred, a2: frood}}`,

pkg/yqlib/operator_entries.go

@@ -0,0 +1,160 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func entrySeqFor(key *yaml.Node, value *yaml.Node) *yaml.Node {
+	var keyKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "key"}
+	var valueKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "value"}
+
+	return &yaml.Node{
+		Kind:    yaml.MappingNode,
+		Tag:     "!!map",
+		Content: []*yaml.Node{keyKey, key, valueKey, value},
+	}
+}
+
+func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {
+	var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+	var entriesNode = candidateNode.CreateChild(nil, sequence)
+
+	var contents = unwrapDoc(candidateNode.Node).Content
+	for index := 0; index < len(contents); index = index + 2 {
+		key := contents[index]
+		value := contents[index+1]
+
+		sequence.Content = append(sequence.Content, entrySeqFor(key, value))
+	}
+	return entriesNode
+}
+
+func toEntriesfromSeq(candidateNode *CandidateNode) *CandidateNode {
+	var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+	var entriesNode = candidateNode.CreateChild(nil, sequence)
+
+	var contents = unwrapDoc(candidateNode.Node).Content
+	for index := 0; index < len(contents); index = index + 1 {
+		key := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!int", Value: fmt.Sprintf("%v", index)}
+		value := contents[index]
+
+		sequence.Content = append(sequence.Content, entrySeqFor(key, value))
+	}
+	return entriesNode
+}
+
+func toEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	var results = list.New()
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+
+		switch candidateNode.Kind {
+		case yaml.MappingNode:
+			results.PushBack(toEntriesFromMap(candidate))
+
+		case yaml.SequenceNode:
+			results.PushBack(toEntriesfromSeq(candidate))
+		default:
+			return Context{}, fmt.Errorf("%v has no keys", candidate.Node.Tag)
+		}
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func parseEntry(d *dataTreeNavigator, entry *yaml.Node, position int) (*yaml.Node, *yaml.Node, error) {
+	prefs := traversePreferences{DontAutoCreate: true}
+	candidateNode := &CandidateNode{Node: entry}
+
+	keyResults, err := traverseMap(Context{}, candidateNode, "key", prefs, false)
+
+	if err != nil {
+		return nil, nil, err
+	} else if keyResults.Len() != 1 {
+		return nil, nil, fmt.Errorf("Expected to find one 'key' entry but found %v in position %v", keyResults.Len(), position)
+	}
+
+	valueResults, err := traverseMap(Context{}, candidateNode, "value", prefs, false)
+
+	if err != nil {
+		return nil, nil, err
+	} else if valueResults.Len() != 1 {
+		return nil, nil, fmt.Errorf("Expected to find one 'value' entry but found %v in position %v", valueResults.Len(), position)
+	}
+
+	return keyResults.Front().Value.(*CandidateNode).Node, valueResults.Front().Value.(*CandidateNode).Node, nil
+
+}
+
+func fromEntries(d *dataTreeNavigator, candidateNode *CandidateNode) (*CandidateNode, error) {
+	var node = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
+	var mapCandidateNode = candidateNode.CreateChild(nil, node)
+
+	var contents = unwrapDoc(candidateNode.Node).Content
+
+	for index := 0; index < len(contents); index = index + 1 {
+		key, value, err := parseEntry(d, contents[index], index)
+		if err != nil {
+			return nil, err
+		}
+
+		node.Content = append(node.Content, key, value)
+	}
+	return mapCandidateNode, nil
+}
+
+func fromEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	var results = list.New()
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+
+		switch candidateNode.Kind {
+		case yaml.SequenceNode:
+			mapResult, err := fromEntries(d, candidate)
+			if err != nil {
+				return Context{}, err
+			}
+			results.PushBack(mapResult)
+		default:
+			return Context{}, fmt.Errorf("from entries only runs against arrays")
+		}
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	//to_entries on the context
+	toEntries, err := toEntriesOperator(d, context, expressionNode)
+	if err != nil {
+		return Context{}, nil
+	}
+
+	//run expression against entries
+	// splat toEntries and pipe it into Rhs
+	splatted, err := splat(d, toEntries, traversePreferences{})
+	if err != nil {
+		return Context{}, nil
+	}
+
+	result, err := d.GetMatchingNodes(splatted, expressionNode.Rhs)
+	log.Debug("expressionNode.Rhs %v", expressionNode.Rhs.Operation.OperationType)
+	log.Debug("result %v", result)
+	if err != nil {
+		return Context{}, nil
+	}
+
+	collected, err := collectOperator(d, result, expressionNode)
+	if err != nil {
+		return Context{}, nil
+	}
+
+	//from_entries on the result
+	return fromEntriesOperator(d, collected, expressionNode)
+}

pkg/yqlib/operator_entries_test.go

@@ -0,0 +1,56 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var entriesOperatorScenarios = []expressionScenario{
+	{
+		description: "to_entries Map",
+		document:    `{a: 1, b: 2}`,
+		expression:  `to_entries`,
+		expected: []string{
+			"D0, P[], (!!seq)::- key: a\n  value: 1\n- key: b\n  value: 2\n",
+		},
+	},
+	{
+		description: "to_entries Array",
+		document:    `[a, b]`,
+		expression:  `to_entries`,
+		expected: []string{
+			"D0, P[], (!!seq)::- key: 0\n  value: a\n- key: 1\n  value: b\n",
+		},
+	},
+	{
+		description: "from_entries map",
+		document:    `{a: 1, b: 2}`,
+		expression:  `to_entries | from_entries`,
+		expected: []string{
+			"D0, P[], (!!map)::a: 1\nb: 2\n",
+		},
+	},
+	{
+		description:    "from_entries with numeric key indexes",
+		subdescription: "from_entries always creates a map, even for numeric keys",
+		document:       `[a,b]`,
+		expression:     `to_entries | from_entries`,
+		expected: []string{
+			"D0, P[], (!!map)::0: a\n1: b\n",
+		},
+	},
+	{
+		description: "Use with_entries to update keys",
+		document:    `{a: 1, b: 2}`,
+		expression:  `with_entries(.key |= "KEY_" + .)`,
+		expected: []string{
+			"D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n",
+		},
+	},
+}
+
+func TestEntriesOperatorScenarios(t *testing.T) {
+	for _, tt := range entriesOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Entries", entriesOperatorScenarios)
+}

pkg/yqlib/operator_equals.go

@@ -4,13 +4,22 @@ import "gopkg.in/yaml.v3"
 
 func equalsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- equalsOperation")
-	return crossFunction(d, context, expressionNode, isEquals(false))
+	return crossFunction(d, context, expressionNode, isEquals(false), true)
 }
 
 func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		value := false
 
+		if lhs == nil && rhs == nil {
+			owner := &CandidateNode{}
+			return createBooleanCandidate(owner, !flip), nil
+		} else if lhs == nil {
+			return createBooleanCandidate(rhs, flip), nil
+		} else if rhs == nil {
+			return createBooleanCandidate(lhs, flip), nil
+		}
+
 		lhsNode := unwrapDoc(lhs.Node)
 		rhsNode := unwrapDoc(rhs.Node)
 
@@ -29,5 +38,5 @@ func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *Candid
 
 func notEqualsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- equalsOperation")
-	return crossFunction(d, context, expressionNode, isEquals(true))
+	return crossFunction(d, context, expressionNode, isEquals(true), true)
 }

pkg/yqlib/operator_equals_test.go

@@ -87,6 +87,22 @@ var equalsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::true\n",
 		},
 	},
+	{
+		description: "Non exisitant key doesn't equal a value",
+		document:    "a: frog",
+		expression:  `select(.b != "thing")`,
+		expected: []string{
+			"D0, P[], (doc)::a: frog\n",
+		},
+	},
+	{
+		description: "Two non existant keys are equal",
+		document:    "a: frog",
+		expression:  `select(.b == .c)`,
+		expected: []string{
+			"D0, P[], (doc)::a: frog\n",
+		},
+	},
 }
 
 func TestEqualOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_multiply.go

@@ -17,7 +17,7 @@ type multiplyPreferences struct {
 
 func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- MultiplyOperator")
-	return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)))
+	return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)), false)
 }
 
 func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

pkg/yqlib/operator_multiply_test.go

@@ -24,6 +24,12 @@ list2:
     - "123"
 `
 
+var mergeArraysObjectKeysText = `It's a complex command, the trickyness comes from needing to have the right context in the expressions.
+First we save the second array into a variable '$two' which lets us reference it later.
+We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. 
+We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter.
+`
+
 var multiplyOperatorScenarios = []expressionScenario{
 	{
 		description: "Multiply integers",
@@ -245,6 +251,16 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!seq)::[{name: fred, age: 34}, {name: bob, age: 32}]\n",
 		},
 	},
+	{
+		description:    "Merge arrays of objects together, matching on a key",
+		subdescription: mergeArraysObjectKeysText,
+		document:       `[{a: apple, b: appleB}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB}]`,
+		document2:      `[{a: banana, c: bananaC}, {a: apple, b: appleB2}, {a: dingo, c: dingoC}]`,
+		expression:     `(select(fi==1) | .[]) as $two | select(fi==0) | .[] |= (. as $cur |  $cur * ($two | select(.a == $cur.a)))`,
+		expected: []string{
+			"D0, P[], (doc)::[{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}]\n",
+		},
+	},
 	{
 		description: "Merge to prefix an element",
 		document:    `{a: cat, b: dog}`,
@@ -283,5 +299,5 @@ func TestMultiplyOperatorScenarios(t *testing.T) {
 	for _, tt := range multiplyOperatorScenarios {
 		testScenario(t, &tt)
 	}
-	documentScenarios(t, "Multiply", multiplyOperatorScenarios)
+	documentScenarios(t, "Multiply (Merge)", multiplyOperatorScenarios)
 }

pkg/yqlib/operator_select_test.go

@@ -14,6 +14,16 @@ var selectOperatorScenarios = []expressionScenario{
 			"D0, P[1], (!!str)::goat\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "a: hello",
+		document2:  "b: world",
+		expression: `select(.a == "hello" or .b == "world")`,
+		expected: []string{
+			"D0, P[], (doc)::a: hello\n",
+			"D0, P[], (doc)::b: world\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `[{animal: cat, legs: {cool: true}}, {animal: fish}]`,

pkg/yqlib/operator_strings.go

@@ -3,11 +3,77 @@ package yqlib
 import (
 	"container/list"
 	"fmt"
+	"regexp"
 	"strings"
 
 	"gopkg.in/yaml.v3"
 )
 
+func getSubstituteParameters(d *dataTreeNavigator, block *ExpressionNode, context Context) (string, string, error) {
+	regEx := ""
+	replacementText := ""
+
+	regExNodes, err := d.GetMatchingNodes(context, block.Lhs)
+	if err != nil {
+		return "", "", err
+	}
+	if regExNodes.MatchingNodes.Front() != nil {
+		regEx = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
+	}
+
+	log.Debug("regEx %v", regEx)
+
+	replacementNodes, err := d.GetMatchingNodes(context, block.Rhs)
+	if err != nil {
+		return "", "", err
+	}
+	if replacementNodes.MatchingNodes.Front() != nil {
+		replacementText = replacementNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
+	}
+
+	return regEx, replacementText, nil
+}
+
+func substitute(original string, regex *regexp.Regexp, replacement string) *yaml.Node {
+	replacedString := regex.ReplaceAllString(original, replacement)
+	return &yaml.Node{Kind: yaml.ScalarNode, Value: replacedString, Tag: "!!str"}
+}
+
+func substituteStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	//rhs  block operator
+	//lhs of block = regex
+	//rhs of block = replacement expression
+	block := expressionNode.Rhs
+
+	regExStr, replacementText, err := getSubstituteParameters(d, block, context)
+
+	if err != nil {
+		return Context{}, err
+	}
+
+	regEx, err := regexp.Compile(regExStr)
+	if err != nil {
+		return Context{}, err
+	}
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		node := unwrapDoc(candidate.Node)
+		if node.Tag != "!!str" {
+			return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation.", node.Tag)
+		}
+
+		targetNode := substitute(node.Value, regEx, replacementText)
+		result := candidate.CreateChild(nil, targetNode)
+		results.PushBack(result)
+	}
+
+	return context.ChildContext(results), nil
+
+}
+
 func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- joinStringOperator")
 	joinStr := ""
@@ -26,7 +92,7 @@ func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *E
 		candidate := el.Value.(*CandidateNode)
 		node := unwrapDoc(candidate.Node)
 		if node.Kind != yaml.SequenceNode {
-			return Context{}, fmt.Errorf("Cannot join with %v, can only join arrays of scalars", node.Tag)
+			return Context{}, fmt.Errorf("cannot join with %v, can only join arrays of scalars", node.Tag)
 		}
 		targetNode := join(node.Content, joinStr)
 		result := candidate.CreateChild(nil, targetNode)

pkg/yqlib/operator_strings_test.go

@@ -13,6 +13,24 @@ var stringsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!str)::cat; meow; 1; ; true\n",
 		},
 	},
+	{
+		description:    "Substitute / Replace string",
+		subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)\nNote the use of `|=` to run in context of the current string value.",
+		document:       `a: dogs are great`,
+		expression:     `.a |= sub("dogs", "cats")`,
+		expected: []string{
+			"D0, P[], (doc)::a: cats are great\n",
+		},
+	},
+	{
+		description:    "Substitute / Replace string with regex",
+		subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)\nNote the use of `|=` to run in context of the current string value.",
+		document:       "a: cat\nb: heat",
+		expression:     `.[] |= sub("(a)", "${1}r")`,
+		expected: []string{
+			"D0, P[], (doc)::a: cart\nb: heart\n",
+		},
+	},
 	{
 		description: "Split strings",
 		document:    `"cat; meow; 1; ; true"`,

pkg/yqlib/operator_style_test.go

@@ -5,6 +5,23 @@ import (
 )
 
 var styleOperatorScenarios = []expressionScenario{
+	{
+		description: "Update and set style of a particular node (simple)",
+		document:    `a: {b: thing, c: something}`,
+		expression:  `.a.b = "new" | .a.b style="double"`,
+		expected: []string{
+			"D0, P[], (doc)::a: {b: \"new\", c: something}\n",
+		},
+	},
+	{
+		description:    "Update and set style of a particular node using path variables",
+		subdescription: "You can use a variable to re-use a path",
+		document:       `a: {b: thing, c: something}`,
+		expression:     `.a.b as $x | $x = "new" | $x style="double"`,
+		expected: []string{
+			"D0, P[], (doc)::a: {b: \"new\", c: something}\n",
+		},
+	},
 	{
 		description: "Set tagged style",
 		document:    `{a: cat, b: 5, c: 3.2, e: true}`,

pkg/yqlib/operator_subtract.go

@@ -25,7 +25,7 @@ func subtractAssignOperator(d *dataTreeNavigator, context Context, expressionNod
 func subtractOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("Subtract operator")
 
-	return crossFunction(d, context, expressionNode, subtract)
+	return crossFunction(d, context, expressionNode, subtract, false)
 }
 
 func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

pkg/yqlib/operator_traverse_path.go

@@ -14,6 +14,7 @@ type traversePreferences struct {
 	IncludeMapKeys       bool
 	DontAutoCreate       bool // by default, we automatically create entries on the fly.
 	DontIncludeMapValues bool
+	OptionalTraverse     bool // e.g. .adf?
 }
 
 func splat(d *dataTreeNavigator, context Context, prefs traversePreferences) (Context, error) {
@@ -60,7 +61,7 @@ func traverse(d *dataTreeNavigator, context Context, matchingNode *CandidateNode
 
 	case yaml.SequenceNode:
 		log.Debug("its a sequence of %v things!", len(value.Content))
-		return traverseArray(matchingNode, operation)
+		return traverseArray(matchingNode, operation, operation.Preferences.(traversePreferences))
 
 	case yaml.AliasNode:
 		log.Debug("its an alias!")
@@ -89,14 +90,19 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
 	// rhs is a collect expression that will yield indexes to retreive of the arrays
 
 	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+
 	if err != nil {
 		return Context{}, err
 	}
+	prefs := traversePreferences{}
 
+	if expressionNode.Rhs.Rhs != nil && expressionNode.Rhs.Rhs.Operation.Preferences != nil {
+		prefs = expressionNode.Rhs.Rhs.Operation.Preferences.(traversePreferences)
+	}
 	var indicesToTraverse = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Content
 
 	//now we traverse the result of the lhs against the indices we found
-	result, err := traverseNodesWithArrayIndices(lhs, indicesToTraverse, traversePreferences{})
+	result, err := traverseNodesWithArrayIndices(lhs, indicesToTraverse, prefs)
 	if err != nil {
 		return Context{}, err
 	}
@@ -130,7 +136,7 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT
 		matchingNode.Node = node.Alias
 		return traverseArrayIndices(context, matchingNode, indicesToTraverse, prefs)
 	} else if node.Kind == yaml.SequenceNode {
-		return traverseArrayWithIndices(matchingNode, indicesToTraverse)
+		return traverseArrayWithIndices(matchingNode, indicesToTraverse, prefs)
 	} else if node.Kind == yaml.MappingNode {
 		return traverseMapWithIndices(context, matchingNode, indicesToTraverse, prefs)
 	} else if node.Kind == yaml.DocumentNode {
@@ -159,7 +165,7 @@ func traverseMapWithIndices(context Context, candidate *CandidateNode, indices [
 	return matchingNodeMap, nil
 }
 
-func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*list.List, error) {
+func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs traversePreferences) (*list.List, error) {
 	log.Debug("traverseArrayWithIndices")
 	var newMatches = list.New()
 	node := unwrapDoc(candidate.Node)
@@ -177,6 +183,9 @@ func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*
 	for _, indexNode := range indices {
 		log.Debug("traverseArrayWithIndices: '%v'", indexNode.Value)
 		index, err := strconv.ParseInt(indexNode.Value, 10, 64)
+		if err != nil && prefs.OptionalTraverse {
+			continue
+		}
 		if err != nil {
 			return nil, fmt.Errorf("Cannot index array with '%v' (%v)", indexNode.Value, err)
 		}
@@ -297,8 +306,8 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *C
 	return nil
 }
 
-func traverseArray(candidate *CandidateNode, operation *Operation) (*list.List, error) {
+func traverseArray(candidate *CandidateNode, operation *Operation, prefs traversePreferences) (*list.List, error) {
 	log.Debug("operation Value %v", operation.Value)
 	indices := []*yaml.Node{&yaml.Node{Value: operation.StringValue}}
-	return traverseArrayWithIndices(candidate, indices)
+	return traverseArrayWithIndices(candidate, indices, prefs)
 }

pkg/yqlib/operator_traverse_path_test.go

@@ -45,6 +45,13 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[1], (!!map)::{c: banana}\n",
 		},
 	},
+	{
+		description:    "Optional Splat",
+		subdescription: "Just like splat, but won't error if you run it against scalars",
+		document:       `"cat"`,
+		expression:     `.[]`,
+		expected:       []string{},
+	},
 	{
 		description:    "Special characters",
 		subdescription: "Use quotes with brackets around path elements with special characters",
@@ -97,6 +104,19 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[a b], (!!null)::null\n",
 		},
 	},
+	{
+		description:    "Optional identifier",
+		subdescription: "Like jq, does not output an error when the yaml is not an array or object as expected",
+		document:       `[1,2,3]`,
+		expression:     `.a?`,
+		expected:       []string{},
+	},
+	{
+		skipDoc:    true,
+		document:   `[[1,2,3], {a: frog}]`,
+		expression: `.[] | .["a"]?`,
+		expected:   []string{"D0, P[1 a], (!!str)::frog\n"},
+	},
 	{
 		skipDoc:    true,
 		document:   ``,

pkg/yqlib/operators.go

@@ -24,7 +24,20 @@ func emptyOperator(d *dataTreeNavigator, context Context, expressionNode *Expres
 
 type crossFunctionCalculation func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)
 
-func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (Context, error) {
+func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *CandidateNode, rhs Context, calculation crossFunctionCalculation, results *list.List) error {
+	for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() {
+		log.Debugf("Applying calc")
+		rhsCandidate := rightEl.Value.(*CandidateNode)
+		resultCandidate, err := calculation(d, context, lhsCandidate, rhsCandidate)
+		if err != nil {
+			return err
+		}
+		results.PushBack(resultCandidate)
+	}
+	return nil
+}
+
+func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation, calcWhenEmpty bool) (Context, error) {
 	var results = list.New()
 	lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
 	if err != nil {
@@ -38,24 +51,33 @@ func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *Expressi
 		return Context{}, err
 	}
 
-	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
-		lhsCandidate := el.Value.(*CandidateNode)
-
-		for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() {
-			log.Debugf("Applying calc")
-			rhsCandidate := rightEl.Value.(*CandidateNode)
-			resultCandidate, err := calculation(d, context, lhsCandidate, rhsCandidate)
+	if calcWhenEmpty && lhs.MatchingNodes.Len() == 0 {
+		if rhs.MatchingNodes.Len() == 0 {
+			resultCandidate, err := calculation(d, context, nil, nil)
 			if err != nil {
 				return Context{}, err
 			}
 			results.PushBack(resultCandidate)
 		}
+		err := resultsForRhs(d, context, nil, rhs, calculation, results)
+		if err != nil {
+			return Context{}, err
+		}
+	}
+
+	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
+		lhsCandidate := el.Value.(*CandidateNode)
+
+		err := resultsForRhs(d, context, lhsCandidate, rhs, calculation, results)
+		if err != nil {
+			return Context{}, err
+		}
 
 	}
 	return context.ChildContext(results), nil
 }
 
-func crossFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (Context, error) {
+func crossFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation, calcWhenEmpty bool) (Context, error) {
 	var results = list.New()
 
 	var evaluateAllTogether = true
@@ -66,11 +88,11 @@ func crossFunction(d *dataTreeNavigator, context Context, expressionNode *Expres
 		}
 	}
 	if evaluateAllTogether {
-		return doCrossFunc(d, context, expressionNode, calculation)
+		return doCrossFunc(d, context, expressionNode, calculation, calcWhenEmpty)
 	}
 
 	for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
-		innerResults, err := doCrossFunc(d, context.SingleChildContext(matchEl.Value.(*CandidateNode)), expressionNode, calculation)
+		innerResults, err := doCrossFunc(d, context.SingleChildContext(matchEl.Value.(*CandidateNode)), expressionNode, calculation, calcWhenEmpty)
 		if err != nil {
 			return Context{}, err
 		}

pkg/yqlib/operators_test.go

@@ -71,7 +71,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 		t.Error(fmt.Errorf("%v: %v", err, s.expression))
 		return
 	}
-	test.AssertResultComplexWithContext(t, s.expected, resultsToString(context.MatchingNodes), fmt.Sprintf("exp: %v\ndoc: %v", s.expression, s.document))
+	test.AssertResultComplexWithContext(t, s.expected, resultsToString(context.MatchingNodes), fmt.Sprintf("desc: %v\nexp: %v\ndoc: %v", s.description, s.expression, s.document))
 }
 
 func resultsToString(results *list.List) []string {

release_instructions.txt

@@ -2,7 +2,7 @@
 - increment version in snapcraft.yaml
 - increment version in github-action/Dockerfile
 - make sure local build passes
-- run gosec (manual because docker platforms)
+- run ./scripts/secure.sh (manual because docker platforms)
 - commit version update changes
 - tag git with same version number
 - commit vX tag - this will trigger github actions

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.6.3'
+version: '4.8.0'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.