Bump version

Fixes #716 #715

cmd/evaluate_all_command.go

@@ -42,14 +42,21 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 		colorsEnabled = true
 	}
 
-	if writeInplace && len(args) < 2 {
+	firstFileIndex := -1
+	if !nullInput && len(args) == 1 {
+		firstFileIndex = 0
+	} else if len(args) > 1 {
+		firstFileIndex = 1
+	}
+
+	if writeInplace && (firstFileIndex == -1) {
 		return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
 	}
 
 	if writeInplace {
 		// only use colors if its forced
 		colorsEnabled = forceColor
-		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[1])
+		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[firstFileIndex])
 		out, err = writeInPlaceHandler.CreateTempFile()
 		if err != nil {
 			return err

cmd/evalute_sequence_command.go

@@ -62,14 +62,21 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		colorsEnabled = true
 	}
 
-	if writeInplace && len(args) < 2 {
+	firstFileIndex := -1
+	if !nullInput && len(args) == 1 {
+		firstFileIndex = 0
+	} else if len(args) > 1 {
+		firstFileIndex = 1
+	}
+
+	if writeInplace && (firstFileIndex == -1) {
 		return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
 	}
 
 	if writeInplace {
 		// only use colors if its forced
 		colorsEnabled = forceColor
-		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[1])
+		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[firstFileIndex])
 		out, err = writeInPlaceHandler.CreateTempFile()
 		if err != nil {
 			return err
@@ -102,7 +109,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer)
 		}
 	default:
-		err = streamEvaluator.EvaluateFiles(args[0], args[1:], printer)
+		err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer)
 	}
 	completedSuccessfully = err == nil
 

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.5.0"
+	Version = "4.6.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.5.0
+FROM mikefarah/yq:4.6.1
 
 COPY entrypoint.sh /entrypoint.sh
 

go.mod

@@ -1,14 +1,14 @@
 module github.com/mikefarah/yq/v4
 
 require (
-	github.com/elliotchance/orderedmap v1.3.0
+	github.com/elliotchance/orderedmap v1.4.0
 	github.com/fatih/color v1.10.0
-	github.com/goccy/go-yaml v1.8.4
-	github.com/jinzhu/copier v0.1.0
+	github.com/goccy/go-yaml v1.8.8
+	github.com/jinzhu/copier v0.2.3
 	github.com/spf13/cobra v1.1.1
 	github.com/timtadh/data-structures v0.5.3 // indirect
 	github.com/timtadh/lexmachine v0.2.2
-	golang.org/x/sys v0.0.0-20210105210732-16f7687f5001 // indirect
+	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

go.sum

@@ -36,8 +36,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/elliotchance/orderedmap v1.3.0 h1:k6m77/d0zCXTjsk12nX40TkEBkSICq8T4s6R6bpCqU0=
-github.com/elliotchance/orderedmap v1.3.0/go.mod h1:8hdSl6jmveQw8ScByd3AaNHNk51RhbTazdqtTty+NFw=
+github.com/elliotchance/orderedmap v1.4.0 h1:wZtfeEONCbx6in1CZyE6bELEt/vFayMvsxqI5SgsR+A=
+github.com/elliotchance/orderedmap v1.4.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
@@ -52,8 +52,8 @@ github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTM
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/goccy/go-yaml v1.8.4 h1:AOEdR7aQgbgwHznGe3BLkDQVujxCPUpHOZZcQcp8Y3M=
-github.com/goccy/go-yaml v1.8.4/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
+github.com/goccy/go-yaml v1.8.8 h1:MGfRB1GeSn/hWXYWS2Pt67iC2GJNnebdIro01ddyucA=
+github.com/goccy/go-yaml v1.8.8/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -101,8 +101,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jinzhu/copier v0.1.0 h1:Vh8xALtH3rrKGB/XIRe5d0yCTHPZFauWPLvdpDAbi88=
-github.com/jinzhu/copier v0.1.0/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
+github.com/jinzhu/copier v0.2.3 h1:Oe09ju+9qft7TffZ7l/04AB2f8u1+V4ZMxmp/nnqeOs=
+github.com/jinzhu/copier v0.2.3/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -182,6 +182,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/timtadh/data-structures v0.5.3 h1:F2tEjoG9qWIyUjbvXVgJqEOGJPMIiYn7U5W5mE+i/vQ=
 github.com/timtadh/data-structures v0.5.3/go.mod h1:9R4XODhJ8JdWFEI8P/HJKqxuJctfBQw6fDibMQny2oU=
@@ -257,8 +259,8 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210105210732-16f7687f5001 h1:/dSxr6gT0FNI1MO5WLJo8mTmItROeOKTkDn+7OwWBos=
-golang.org/x/sys v0.0.0-20210105210732-16f7687f5001/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -324,6 +326,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

pkg/yqlib/candidate_node.go

@@ -2,8 +2,6 @@ package yqlib
 
 import (
 	"fmt"
-	"strconv"
-	"strings"
 
 	"github.com/jinzhu/copier"
 	yaml "gopkg.in/yaml.v3"
@@ -18,10 +16,15 @@ type CandidateNode struct {
 	// when performing op against all nodes given, this will treat all the nodes as one
 	// (e.g. top level cross document merge). This property does not propegate to child nodes.
 	EvaluateTogether bool
+	IsMapKey         bool
 }
 
 func (n *CandidateNode) GetKey() string {
-	return fmt.Sprintf("%v - %v", n.Document, n.Path)
+	keyPrefix := ""
+	if n.IsMapKey {
+		keyPrefix = "key-"
+	}
+	return fmt.Sprintf("%v%v - %v", keyPrefix, n.Document, n.Path)
 }
 
 func (n *CandidateNode) CreateChild(path interface{}, node *yaml.Node) *CandidateNode {
@@ -63,11 +66,10 @@ func (n *CandidateNode) UpdateFrom(other *CandidateNode) {
 	n.UpdateAttributesFrom(other)
 	n.Node.Content = other.Node.Content
 	n.Node.Value = other.Node.Value
-	n.Node.Alias = other.Node.Alias
-	n.Node.Anchor = other.Node.Anchor
 }
 
 func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) {
+	log.Debug("UpdateAttributesFrom: n: %v other: %v", n.GetKey(), other.GetKey())
 	if n.Node.Kind != other.Node.Kind {
 		// clear out the contents when switching to a different type
 		// e.g. map to array
@@ -76,6 +78,8 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) {
 	}
 	n.Node.Kind = other.Node.Kind
 	n.Node.Tag = other.Node.Tag
+	n.Node.Alias = other.Node.Alias
+	n.Node.Anchor = other.Node.Anchor
 
 	// merge will pickup the style of the new thing
 	// when autocreating nodes
@@ -86,46 +90,3 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) {
 	n.Node.HeadComment = n.Node.HeadComment + other.Node.HeadComment
 	n.Node.LineComment = n.Node.LineComment + other.Node.LineComment
 }
-
-func (n *CandidateNode) PathStackToString() string {
-	return mergePathStackToString(n.Path)
-}
-
-func mergePathStackToString(pathStack []interface{}) string {
-	var sb strings.Builder
-	for index, path := range pathStack {
-		switch path.(type) {
-		case int, int64:
-			// if arrayMergeStrategy == AppendArrayMergeStrategy {
-			// sb.WriteString("[+]")
-			// } else {
-			sb.WriteString(fmt.Sprintf("[%v]", path))
-			// }
-
-		default:
-			s := fmt.Sprintf("%v", path)
-			var _, errParsingInt = strconv.ParseInt(s, 10, 64) // nolint
-
-			hasSpecial := strings.Contains(s, ".") || strings.Contains(s, "[") || strings.Contains(s, "]") || strings.Contains(s, "\"")
-			hasDoubleQuotes := strings.Contains(s, "\"")
-			wrappingCharacterStart := "\""
-			wrappingCharacterEnd := "\""
-			if hasDoubleQuotes {
-				wrappingCharacterStart = "("
-				wrappingCharacterEnd = ")"
-			}
-			if hasSpecial || errParsingInt == nil {
-				sb.WriteString(wrappingCharacterStart)
-			}
-			sb.WriteString(s)
-			if hasSpecial || errParsingInt == nil {
-				sb.WriteString(wrappingCharacterEnd)
-			}
-		}
-
-		if index < len(pathStack)-1 {
-			sb.WriteString(".")
-		}
-	}
-	return sb.String()
-}

pkg/yqlib/context.go

@@ -42,3 +42,13 @@ func (n *Context) ChildContext(results *list.List) Context {
 	clone.MatchingNodes = results
 	return clone
 }
+
+func (n *Context) Clone() Context {
+	clone := Context{}
+	err := copier.Copy(&clone, n)
+	if err != nil {
+		log.Error("Error cloning context :(")
+		panic(err)
+	}
+	return clone
+}

pkg/yqlib/doc/Equals.md

@@ -29,6 +29,24 @@ true
 false
 ```
 
+## Don't match string
+Given a sample.yml file of:
+```yaml
+- cat
+- goat
+- dog
+```
+then
+```bash
+yq eval '.[] | (. != "*at")' sample.yml
+```
+will output
+```yaml
+false
+false
+true
+```
+
 ## Match number
 Given a sample.yml file of:
 ```yaml
@@ -47,6 +65,24 @@ true
 false
 ```
 
+## Dont match number
+Given a sample.yml file of:
+```yaml
+- 3
+- 4
+- 5
+```
+then
+```bash
+yq eval '.[] | (. != 4)' sample.yml
+```
+will output
+```yaml
+true
+false
+true
+```
+
 ## Match nulls
 Running
 ```bash

pkg/yqlib/doc/File Operators.md

@@ -35,6 +35,26 @@ will output
 0
 ```
 
+## Get file indices of multiple documents
+Given a sample.yml file of:
+```yaml
+a: cat
+```
+And another sample another.yml file of:
+```yaml
+a: cat
+```
+then
+```bash
+yq eval-all 'fileIndex' sample.yml another.yml
+```
+will output
+```yaml
+0
+---
+1
+```
+
 ## Get file index alias
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Multiply.md

@@ -1,19 +1,34 @@
-Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
+Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.
 
-To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
+## Objects and arrays - merging
+Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged.
 
-To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?.
 Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
 
-Multiplication of strings and numbers are not yet supported.
+### Merge Flags
+You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`.  See examples below
 
-## Merging files
+- `+` to append arrays
+- `?` to only merge existing fields
+- `d` to deeply merge arrays
+
+### Merging files
 Note the use of `eval-all` to ensure all documents are loaded into memory.
 
 ```bash
 yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml
 ```
 
+## Multiply integers
+Running
+```bash
+yq eval --null-input '3 * 4'
+```
+will output
+```yaml
+12
+```
+
 ## Merge objects together, returning merged result only
 Given a sample.yml file of:
 ```yaml
@@ -71,7 +86,6 @@ Given a sample.yml file of:
 a: {things: great}
 b:
   also: "me"
-
 ```
 then
 ```bash
@@ -191,6 +205,32 @@ thing:
   - 4
 ```
 
+## Merge, deeply merging arrays
+Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.
+
+Given a sample.yml file of:
+```yaml
+a:
+  - name: fred
+    age: 12
+  - name: bob
+    age: 32
+b:
+  - name: fred
+    age: 34
+```
+then
+```bash
+yq eval '.a *d .b' sample.yml
+```
+will output
+```yaml
+- name: fred
+  age: 34
+- name: bob
+  age: 32
+```
+
 ## Merge to prefix an element
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Reduce.md

@@ -0,0 +1,76 @@
+Reduce is a powerful way to process a collection of data into a new form.
+
+```
+<exp> as $<name> ireduce (<init>; <block>)
+```
+
+e.g.
+
+```
+.[] as $item ireduce (0; . + $item)
+```
+
+On the LHS we are configuring the collection of items that will be reduced `<exp>` as well as what each element will be called `$<name>`. Note that the array has been splatted into its individual elements.
+
+On the RHS there is `<init>`, the starting value of the accumulator and `<block>`, the expression that will update the accumulator for each element in the collection. Note that within the block expression, `.` will evaluate to the current value of the accumulator. 
+
+## yq vs jq syntax
+Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`).
+
+To that end, the reduce operator is called `ireduce` for backwards compatability if a `jq` like prefix version of `reduce` is ever added.
+
+
+## Sum numbers
+Given a sample.yml file of:
+```yaml
+- 10
+- 2
+- 5
+- 3
+```
+then
+```bash
+yq eval '.[] as $item ireduce (0; . + $item)' sample.yml
+```
+will output
+```yaml
+20
+```
+
+## Merge all yaml files together
+Given a sample.yml file of:
+```yaml
+a: cat
+```
+And another sample another.yml file of:
+```yaml
+b: dog
+```
+then
+```bash
+yq eval-all '. as $item ireduce ({}; . * $item )' sample.yml another.yml
+```
+will output
+```yaml
+a: cat
+b: dog
+```
+
+## Convert an array to an object
+Given a sample.yml file of:
+```yaml
+- name: Cathy
+  has: apples
+- name: Bob
+  has: bananas
+```
+then
+```bash
+yq eval '.[] as $item ireduce ({}; .[$item | .name] = ($item | .has) )' sample.yml
+```
+will output
+```yaml
+Cathy: apples
+Bob: bananas
+```
+

pkg/yqlib/doc/Traverse (Read).md

@@ -33,7 +33,7 @@ c: banana
 ```
 
 ## Special characters
-Use quotes around path elements with special characters
+Use quotes with brackets around path elements with special characters
 
 Given a sample.yml file of:
 ```yaml
@@ -41,7 +41,23 @@ Given a sample.yml file of:
 ```
 then
 ```bash
-yq eval '."{}"' sample.yml
+yq eval '.["{}"]' sample.yml
+```
+will output
+```yaml
+frog
+```
+
+## Keys with spaces
+Use quotes with brackets around path elements with special characters
+
+Given a sample.yml file of:
+```yaml
+"red rabbit": frog
+```
+then
+```bash
+yq eval '.["red rabbit"]' sample.yml
 ```
 will output
 ```yaml

pkg/yqlib/doc/headers/Multiply.md

@@ -1,13 +1,18 @@
-Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
+Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.
 
-To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
+## Objects and arrays - merging
+Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged.
 
-To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?.
 Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
 
-Multiplication of strings and numbers are not yet supported.
+### Merge Flags
+You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`.  See examples below
 
-## Merging files
+- `+` to append arrays
+- `?` to only merge existing fields
+- `d` to deeply merge arrays
+
+### Merging files
 Note the use of `eval-all` to ensure all documents are loaded into memory.
 
 ```bash

pkg/yqlib/doc/headers/Reduce.md

@@ -0,0 +1,21 @@
+Reduce is a powerful way to process a collection of data into a new form.
+
+```
+<exp> as $<name> ireduce (<init>; <block>)
+```
+
+e.g.
+
+```
+.[] as $item ireduce (0; . + $item)
+```
+
+On the LHS we are configuring the collection of items that will be reduced `<exp>` as well as what each element will be called `$<name>`. Note that the array has been splatted into its individual elements.
+
+On the RHS there is `<init>`, the starting value of the accumulator and `<block>`, the expression that will update the accumulator for each element in the collection. Note that within the block expression, `.` will evaluate to the current value of the accumulator. 
+
+## yq vs jq syntax
+Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`).
+
+To that end, the reduce operator is called `ireduce` for backwards compatability if a `jq` like prefix version of `reduce` is ever added.
+

pkg/yqlib/expression_parser.go

@@ -5,9 +5,6 @@ import (
 	"strings"
 )
 
-var myPathTokeniser = newExpressionTokeniser()
-var myPathPostfixer = newExpressionPostFixer()
-
 type ExpressionNode struct {
 	Operation *Operation
 	Lhs       *ExpressionNode
@@ -19,19 +16,21 @@ type ExpressionParser interface {
 }
 
 type expressionParserImpl struct {
+	pathTokeniser expressionTokeniser
+	pathPostFixer expressionPostFixer
 }
 
 func NewExpressionParser() ExpressionParser {
-	return &expressionParserImpl{}
+	return &expressionParserImpl{newExpressionTokeniser(), newExpressionPostFixer()}
 }
 
 func (p *expressionParserImpl) ParseExpression(expression string) (*ExpressionNode, error) {
-	tokens, err := myPathTokeniser.Tokenise(expression)
+	tokens, err := p.pathTokeniser.Tokenise(expression)
 	if err != nil {
 		return nil, err
 	}
 	var Operations []*Operation
-	Operations, err = myPathPostfixer.ConvertToPostfix(tokens)
+	Operations, err = p.pathPostFixer.ConvertToPostfix(tokens)
 	if err != nil {
 		return nil, err
 	}

pkg/yqlib/expression_processing_test.go

@@ -12,6 +12,11 @@ var pathTests = []struct {
 	expectedTokens  []interface{}
 	expectedPostFix []interface{}
 }{
+	{
+		`.a as $item ireduce (0; . + $item)`, // note - add code to shuffle reduce to this position for postfix
+		append(make([]interface{}, 0), "a", "ASSIGN_VARIABLE", "GET_VARIABLE", "REDUCE", "(", "0 (int64)", "BLOCK", "SELF", "ADD", "GET_VARIABLE", ")"),
+		append(make([]interface{}, 0), "a", "GET_VARIABLE", "ASSIGN_VARIABLE", "0 (int64)", "SELF", "GET_VARIABLE", "ADD", "BLOCK", "REDUCE"),
+	},
 	{
 		`.a | .b | .c`,
 		append(make([]interface{}, 0), "a", "PIPE", "b", "PIPE", "c"),

pkg/yqlib/expression_tokeniser.go

@@ -100,6 +100,9 @@ func multiplyWithPrefs() lex.Action {
 		if strings.Contains(options, "?") {
 			prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
 		}
+		if strings.Contains(options, "d") {
+			prefs.DeepMergeArrays = true
+		}
 		op := &Operation{OperationType: multiplyOpType, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs}
 		return &token{TokenType: operationToken, Operation: op}, nil
 	}
@@ -186,7 +189,7 @@ func getVariableOpToken() lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		value := string(m.Bytes)
 
-		value = value[1 : len(value)-1]
+		value = value[1:]
 
 		getVarOperation := createValueOperation(value, value)
 		getVarOperation.OperationType = getVariableOpType
@@ -252,6 +255,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`or`), opToken(orOpType))
 	lexer.Add([]byte(`and`), opToken(andOpType))
 	lexer.Add([]byte(`not`), opToken(notOpType))
+	lexer.Add([]byte(`ireduce`), opToken(reduceOpType))
+	lexer.Add([]byte(`;`), opToken(blockOpType))
 	lexer.Add([]byte(`\/\/`), opToken(alternativeOpType))
 
 	lexer.Add([]byte(`documentIndex`), opToken(getDocumentIndexOpType))
@@ -284,6 +289,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`collect`), opToken(collectOpType))
 
 	lexer.Add([]byte(`\s*==\s*`), opToken(equalsOpType))
+	lexer.Add([]byte(`\s*!=\s*`), opToken(notEqualsOpType))
 	lexer.Add([]byte(`\s*=\s*`), assignOpToken(false))
 
 	lexer.Add([]byte(`del`), opToken(deleteChildOpType))
@@ -316,13 +322,13 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`\]`), literalToken(closeCollect, true))
 	lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
 	lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
-	lexer.Add([]byte(`\*[\+|\?]*`), multiplyWithPrefs())
+	lexer.Add([]byte(`\*[\+|\?d]*`), multiplyWithPrefs())
 	lexer.Add([]byte(`\+`), opToken(addOpType))
 	lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
 	lexer.Add([]byte(`\$[a-zA-Z_-0-9]+`), getVariableOpToken())
 	lexer.Add([]byte(`as`), opToken(assignVariableOpType))
 
-	err := lexer.Compile()
+	err := lexer.CompileNFA()
 	if err != nil {
 		return nil, err
 	}

pkg/yqlib/lib.go

@@ -25,6 +25,9 @@ type operationType struct {
 
 var orOpType = &operationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: orOperator}
 var andOpType = &operationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: andOperator}
+var reduceOpType = &operationType{Type: "REDUCE", NumArgs: 2, Precedence: 35, Handler: reduceOperator}
+
+var blockOpType = &operationType{Type: "BLOCK", Precedence: 10, NumArgs: 2, Handler: emptyOperator}
 
 var unionOpType = &operationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: unionOperator}
 
@@ -46,6 +49,7 @@ var addOpType = &operationType{Type: "ADD", NumArgs: 2, Precedence: 42, Handler:
 var alternativeOpType = &operationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 42, Handler: alternativeOperator}
 
 var equalsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: equalsOperator}
+var notEqualsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: notEqualsOperator}
 var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 40, Handler: createMapOperator}
 
 var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: 45, Handler: pipeOperator}
@@ -79,7 +83,7 @@ var selfReferenceOpType = &operationType{Type: "SELF", NumArgs: 0, Precedence: 5
 var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: valueOperator}
 var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator}
 var notOpType = &operationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: notOperator}
-var emptyOpType = &operationType{Type: "EMPTY", NumArgs: 50, Handler: emptyOperator}
+var emptyOpType = &operationType{Type: "EMPTY", Precedence: 50, Handler: emptyOperator}
 
 var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: recursiveDescentOperator}
 

pkg/yqlib/operator_assign.go

@@ -36,6 +36,7 @@ func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode
 
 // does not update content or values
 func assignAttributesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debug("getting lhs matching nodes for update")
 	lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
 	if err != nil {
 		return Context{}, err

pkg/yqlib/operator_delete.go

@@ -7,8 +7,9 @@ import (
 )
 
 func deleteChildOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-
-	nodesToDelete, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+	contextToUse := context.Clone()
+	contextToUse.DontAutoCreate = true
+	nodesToDelete, err := d.GetMatchingNodes(contextToUse, expressionNode.Rhs)
 
 	if err != nil {
 		return Context{}, err
@@ -17,19 +18,21 @@ func deleteChildOperator(d *dataTreeNavigator, context Context, expressionNode *
 	for el := nodesToDelete.MatchingNodes.Back(); el != nil; el = el.Prev() {
 		candidate := el.Value.(*CandidateNode)
 
-		deleteImmediateChildOp := &Operation{
-			OperationType: deleteImmediateChildOpType,
-			Value:         candidate.Path[len(candidate.Path)-1],
-		}
+		if len(candidate.Path) > 0 {
+			deleteImmediateChildOp := &Operation{
+				OperationType: deleteImmediateChildOpType,
+				Value:         candidate.Path[len(candidate.Path)-1],
+			}
 
-		deleteImmediateChildOpNode := &ExpressionNode{
-			Operation: deleteImmediateChildOp,
-			Rhs:       createTraversalTree(candidate.Path[0:len(candidate.Path)-1], traversePreferences{}),
-		}
+			deleteImmediateChildOpNode := &ExpressionNode{
+				Operation: deleteImmediateChildOp,
+				Rhs:       createTraversalTree(candidate.Path[0:len(candidate.Path)-1], traversePreferences{}, false),
+			}
 
-		_, err := d.GetMatchingNodes(context, deleteImmediateChildOpNode)
-		if err != nil {
-			return Context{}, err
+			_, err := d.GetMatchingNodes(contextToUse, deleteImmediateChildOpNode)
+			if err != nil {
+				return Context{}, err
+			}
 		}
 	}
 	return context, nil

pkg/yqlib/operator_delete_test.go

@@ -53,6 +53,22 @@ var deleteOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: [x, x]\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `a: {thing1: yep, thing2: cool, thing3: hi, b: {thing1: cool, great: huh}}`,
+		expression: `del(..)`,
+		expected: []string{
+			"D0, P[], (!!map)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: {thing1: yep, thing2: cool, thing3: hi, b: {thing1: cool, great: huh}}`,
+		expression: `del(.. | select(tag == "!!map") | (.b.thing1,.thing2))`,
+		expected: []string{
+			"D0, P[], (!!map)::a: {thing1: yep, thing3: hi, b: {great: huh}}\n",
+		},
+	},
 	{
 		description: "Delete nested entry in array",
 		document:    `[{a: cat, b: dog}]`,

pkg/yqlib/operator_equals.go

@@ -1,21 +1,33 @@
 package yqlib
 
+import "gopkg.in/yaml.v3"
+
 func equalsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- equalsOperation")
-	return crossFunction(d, context, expressionNode, isEquals)
+	return crossFunction(d, context, expressionNode, isEquals(false))
 }
 
-func isEquals(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-	value := false
+func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+		value := false
 
-	lhsNode := unwrapDoc(lhs.Node)
-	rhsNode := unwrapDoc(rhs.Node)
+		lhsNode := unwrapDoc(lhs.Node)
+		rhsNode := unwrapDoc(rhs.Node)
 
-	if lhsNode.Tag == "!!null" {
-		value = (rhsNode.Tag == "!!null")
-	} else {
-		value = matchKey(lhsNode.Value, rhsNode.Value)
+		if lhsNode.Tag == "!!null" {
+			value = (rhsNode.Tag == "!!null")
+		} else if lhsNode.Kind == yaml.ScalarNode && rhsNode.Kind == yaml.ScalarNode {
+			value = matchKey(lhsNode.Value, rhsNode.Value)
+		}
+		log.Debugf("%v == %v ? %v", NodeToString(lhs), NodeToString(rhs), value)
+		if flip {
+			value = !value
+		}
+		return createBooleanCandidate(lhs, value), nil
 	}
-	log.Debugf("%v == %v ? %v", NodeToString(lhs), NodeToString(rhs), value)
-	return createBooleanCandidate(lhs, value), nil
+}
+
+func notEqualsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("-- equalsOperation")
+	return crossFunction(d, context, expressionNode, isEquals(true))
 }

pkg/yqlib/operator_equals_test.go

@@ -14,6 +14,14 @@ var equalsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::false\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "{a: { b: {things: \"\"}, f: [1], g: [] }}",
+		expression: ".. | select(. == \"\")",
+		expected: []string{
+			"D0, P[a b things], (!!str)::\"\"\n",
+		},
+	},
 	{
 		description: "Match string",
 		document:    `[cat,goat,dog]`,
@@ -23,7 +31,18 @@ var equalsOperatorScenarios = []expressionScenario{
 			"D0, P[1], (!!bool)::true\n",
 			"D0, P[2], (!!bool)::false\n",
 		},
-	}, {
+	},
+	{
+		description: "Don't match string",
+		document:    `[cat,goat,dog]`,
+		expression:  `.[] | (. != "*at")`,
+		expected: []string{
+			"D0, P[0], (!!bool)::false\n",
+			"D0, P[1], (!!bool)::false\n",
+			"D0, P[2], (!!bool)::true\n",
+		},
+	},
+	{
 		description: "Match number",
 		document:    `[3, 4, 5]`,
 		expression:  `.[] | (. == 4)`,
@@ -32,7 +51,18 @@ var equalsOperatorScenarios = []expressionScenario{
 			"D0, P[1], (!!bool)::true\n",
 			"D0, P[2], (!!bool)::false\n",
 		},
-	}, {
+	},
+	{
+		description: "Dont match number",
+		document:    `[3, 4, 5]`,
+		expression:  `.[] | (. != 4)`,
+		expected: []string{
+			"D0, P[0], (!!bool)::true\n",
+			"D0, P[1], (!!bool)::false\n",
+			"D0, P[2], (!!bool)::true\n",
+		},
+	},
+	{
 		skipDoc:    true,
 		document:   `a: { cat: {b: apple, c: whatever}, pat: {b: banana} }`,
 		expression: `.a | (.[].b == "apple")`,

pkg/yqlib/operator_file_test.go

@@ -21,6 +21,16 @@ var fileOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!int)::0\n",
 		},
 	},
+	{
+		description: "Get file indices of multiple documents",
+		document:    `{a: cat}`,
+		document2:   `{a: cat}`,
+		expression:  `fileIndex`,
+		expected: []string{
+			"D0, P[], (!!int)::0\n",
+			"D0, P[], (!!int)::1\n",
+		},
+	},
 	{
 		description: "Get file index alias",
 		document:    `{a: cat}`,

pkg/yqlib/operator_multiply.go

@@ -10,8 +10,9 @@ import (
 )
 
 type multiplyPreferences struct {
-	AppendArrays  bool
-	TraversePrefs traversePreferences
+	AppendArrays    bool
+	DeepMergeArrays bool
+	TraversePrefs   traversePreferences
 }
 
 func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@@ -37,11 +38,31 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 			return mergeObjects(d, context, newThing, rhs, preferences)
 		} else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" {
 			return multiplyIntegers(lhs, rhs)
+		} else if (lhs.Node.Tag == "!!int" || lhs.Node.Tag == "!!float") && (rhs.Node.Tag == "!!int" || rhs.Node.Tag == "!!float") {
+			return multiplyFloats(lhs, rhs)
 		}
 		return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
 	}
 }
 
+func multiplyFloats(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	target := lhs.CreateChild(nil, &yaml.Node{})
+	target.Node.Kind = yaml.ScalarNode
+	target.Node.Style = lhs.Node.Style
+	target.Node.Tag = "!!float"
+
+	lhsNum, err := strconv.ParseFloat(lhs.Node.Value, 64)
+	if err != nil {
+		return nil, err
+	}
+	rhsNum, err := strconv.ParseFloat(rhs.Node.Value, 64)
+	if err != nil {
+		return nil, err
+	}
+	target.Node.Value = fmt.Sprintf("%v", lhsNum*rhsNum)
+	return target, nil
+}
+
 func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	target := lhs.CreateChild(nil, &yaml.Node{})
 	target.Node.Kind = yaml.ScalarNode
@@ -66,7 +87,7 @@ func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs
 
 	// shouldn't recurse arrays if appending
 	prefs := recursiveDescentPreferences{RecurseArray: !shouldAppendArrays,
-		TraversePreferences: traversePreferences{DontFollowAlias: true}}
+		TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}}
 	err := recursiveDecent(d, results, context.SingleChildContext(rhs), prefs)
 	if err != nil {
 		return nil, err
@@ -78,7 +99,12 @@ func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs
 	}
 
 	for el := results.Front(); el != nil; el = el.Next() {
-		err := applyAssignment(d, context, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), preferences)
+		candidate := el.Value.(*CandidateNode)
+		if candidate.Node.Tag == "!!merge" {
+			continue
+		}
+
+		err := applyAssignment(d, context, pathIndexToStartFrom, lhs, candidate, preferences)
 		if err != nil {
 			return nil, err
 		}
@@ -88,20 +114,21 @@ func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs
 
 func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) error {
 	shouldAppendArrays := preferences.AppendArrays
-	log.Debugf("merge - applyAssignment lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs))
+	log.Debugf("merge - applyAssignment lhs %v, rhs: %v", lhs.GetKey(), rhs.GetKey())
 
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
 
 	assignmentOp := &Operation{OperationType: assignAttributesOpType}
-	if rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode {
+	if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
+		assignmentOp.OperationType = addAssignOpType
+	} else if !preferences.DeepMergeArrays && rhs.Node.Kind == yaml.SequenceNode ||
+		(rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode) {
 		assignmentOp.OperationType = assignOpType
 		assignmentOp.UpdateAssign = false
-	} else if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
-		assignmentOp.OperationType = addAssignOpType
 	}
 	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
 
-	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs), Rhs: &ExpressionNode{Operation: rhsOp}}
+	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey), Rhs: &ExpressionNode{Operation: rhsOp}}
 
 	_, err := d.GetMatchingNodes(context.SingleChildContext(lhs), assignmentOpNode)
 

pkg/yqlib/operator_multiply_test.go

@@ -5,6 +5,27 @@ import (
 )
 
 var multiplyOperatorScenarios = []expressionScenario{
+	{
+		description: "Multiply integers",
+		expression:  `3 * 4`,
+		expected: []string{
+			"D0, P[], (!!int)::12\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `3 * 4.5`,
+		expected: []string{
+			"D0, P[], (!!float)::13.5\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `4.5 * 3`,
+		expected: []string{
+			"D0, P[], (!!float)::13.5\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `{a: {also: [1]}, b: {also: me}}`,
@@ -13,6 +34,30 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "# b\nb:\n  # a\n  a: cat",
+		expression: "{} * .",
+		expected: []string{
+			"D0, P[], (!!map)::# b\nb:\n    # a\n    a: cat\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "# b\nb:\n  # a\n  a: cat",
+		expression: ". * {}",
+		expected: []string{
+			"D0, P[], (!!map)::# b\nb:\n    # a\n    a: cat\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: &a { b: &b { c: &c cat } } }`,
+		expression: `{} * .`,
+		expected: []string{
+			"D0, P[], (!!map)::{a: &a {b: &b {c: &c cat}}}\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `{a: 2, b: 5}`,
@@ -92,7 +137,7 @@ var multiplyOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{a: {things: great}, b: {also: me}}`,
-		expression: `. * {"a":.b}`,
+		expression: `. * {"a": .b}`,
 		expected: []string{
 			"D0, P[], (!!map)::{a: {things: great, also: me}, b: {also: me}}\n",
 		},
@@ -100,16 +145,10 @@ var multiplyOperatorScenarios = []expressionScenario{
 	{
 		description:           "Merge keeps style of LHS",
 		dontFormatInputForDoc: true,
-		document: `a: {things: great}
-b:
-  also: "me"
-`,
-		expression: `. * {"a":.b}`,
+		document:              "a: {things: great}\nb:\n  also: \"me\"",
+		expression:            `. * {"a":.b}`,
 		expected: []string{
-			`D0, P[], (!!map)::a: {things: great, also: "me"}
-b:
-    also: "me"
-`,
+			"D0, P[], (!!map)::a: {things: great, also: \"me\"}\nb:\n    also: \"me\"\n",
 		},
 	},
 	{
@@ -139,7 +178,7 @@ b:
 	{
 		skipDoc:    true,
 		document:   `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,
-		expression: `.a *? .b`,
+		expression: `.a *?d .b`,
 		expected: []string{
 			"D0, P[a], (!!seq)::[{thing: two}]\n",
 		},
@@ -168,6 +207,15 @@ b:
 			"D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n",
 		},
 	},
+	{
+		description:    "Merge, deeply merging arrays",
+		subdescription: "Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.",
+		document:       `{a: [{name: fred, age: 12}, {name: bob, age: 32}], b: [{name: fred, age: 34}]}`,
+		expression:     `.a *d .b`,
+		expected: []string{
+			"D0, P[a], (!!seq)::[{name: fred, age: 34}, {name: bob, age: 32}]\n",
+		},
+	},
 	{
 		description: "Merge to prefix an element",
 		document:    `{a: cat, b: dog}`,

pkg/yqlib/operator_reduce.go

@@ -0,0 +1,59 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+)
+
+func reduceOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("-- reduceOp")
+	//.a as $var reduce (0; . + $var)
+	//lhs is the assignment operator
+	//rhs is the reduce block
+	// '.' refers to the current accumulator, initialised to 0
+	// $var references a single element from the .a
+
+	//ensure lhs is actually an assignment
+	//and rhs is a block (empty)
+	if expressionNode.Lhs.Operation.OperationType != assignVariableOpType {
+		return Context{}, fmt.Errorf("reduce must be given a variables assignment, got %v instead", expressionNode.Lhs.Operation.OperationType.Type)
+	} else if expressionNode.Rhs.Operation.OperationType != blockOpType {
+		return Context{}, fmt.Errorf("reduce must be given a block, got %v instead", expressionNode.Rhs.Operation.OperationType.Type)
+	}
+
+	arrayExpNode := expressionNode.Lhs.Lhs
+	array, err := d.GetMatchingNodes(context, arrayExpNode)
+
+	log.Debugf("array of %v things", array.MatchingNodes.Len())
+
+	if err != nil {
+		return Context{}, err
+	}
+
+	variableName := expressionNode.Lhs.Rhs.Operation.StringValue
+
+	initExp := expressionNode.Rhs.Lhs
+
+	accum, err := d.GetMatchingNodes(context, initExp)
+	if err != nil {
+		return Context{}, err
+	}
+
+	log.Debugf("with variable %v", variableName)
+
+	blockExp := expressionNode.Rhs.Rhs
+	for el := array.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		log.Debugf("REDUCING WITH %v", NodeToString(candidate))
+		l := list.New()
+		l.PushBack(candidate)
+		accum.SetVariable(variableName, l)
+
+		accum, err = d.GetMatchingNodes(accum, blockExp)
+		if err != nil {
+			return Context{}, err
+		}
+	}
+
+	return accum, nil
+}

pkg/yqlib/operator_reduce_test.go

@@ -0,0 +1,40 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var reduceOperatorScenarios = []expressionScenario{
+	{
+		description: "Sum numbers",
+		document:    `[10,2, 5, 3]`,
+		expression:  `.[] as $item ireduce (0; . + $item)`,
+		expected: []string{
+			"D0, P[], (!!int)::20\n",
+		},
+	},
+	{
+		description: "Merge all yaml files together",
+		document:    `a: cat`,
+		document2:   `b: dog`,
+		expression:  `. as $item ireduce ({}; . * $item )`,
+		expected: []string{
+			"D0, P[], (!!map)::a: cat\nb: dog\n",
+		},
+	},
+	{
+		description: "Convert an array to an object",
+		document:    `[{name: Cathy, has: apples},{name: Bob, has: bananas}]`,
+		expression:  `.[] as $item ireduce ({}; .[$item | .name] = ($item | .has) )`,
+		expected: []string{
+			"D0, P[], (!!map)::Cathy: apples\nBob: bananas\n",
+		},
+	},
+}
+
+func TestReduceOperatorScenarios(t *testing.T) {
+	for _, tt := range reduceOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Reduce", reduceOperatorScenarios)
+}

pkg/yqlib/operator_traverse_path.go

@@ -10,9 +10,10 @@ import (
 )
 
 type traversePreferences struct {
-	DontFollowAlias bool
-	IncludeMapKeys  bool
-	DontAutoCreate  bool // by default, we automatically create entries on the fly.
+	DontFollowAlias      bool
+	IncludeMapKeys       bool
+	DontAutoCreate       bool // by default, we automatically create entries on the fly.
+	DontIncludeMapValues bool
 }
 
 func splat(d *dataTreeNavigator, context Context, prefs traversePreferences) (Context, error) {
@@ -214,10 +215,21 @@ func traverseMap(context Context, matchingNode *CandidateNode, key string, prefs
 	if !prefs.DontAutoCreate && !context.DontAutoCreate && newMatches.Len() == 0 {
 		//no matches, create one automagically
 		valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}
+		keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
 		node := matchingNode.Node
-		node.Content = append(node.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: key}, valueNode)
-		candidateNode := matchingNode.CreateChild(key, valueNode)
-		newMatches.Set(candidateNode.GetKey(), candidateNode)
+		node.Content = append(node.Content, keyNode, valueNode)
+
+		if prefs.IncludeMapKeys {
+			log.Debug("including key")
+			candidateNode := matchingNode.CreateChild(key, keyNode)
+			candidateNode.IsMapKey = true
+			newMatches.Set(fmt.Sprintf("keyOf-%v", candidateNode.GetKey()), candidateNode)
+		}
+		if !prefs.DontIncludeMapValues {
+			log.Debug("including value")
+			candidateNode := matchingNode.CreateChild(key, valueNode)
+			newMatches.Set(candidateNode.GetKey(), candidateNode)
+		}
 	}
 
 	results := list.New()
@@ -253,11 +265,16 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode,
 		} else if splat || keyMatches(key, wantedKey) {
 			log.Debug("MATCHED")
 			if prefs.IncludeMapKeys {
+				log.Debug("including key")
 				candidateNode := candidate.CreateChild(key.Value, key)
+				candidateNode.IsMapKey = true
 				newMatches.Set(fmt.Sprintf("keyOf-%v", candidateNode.GetKey()), candidateNode)
 			}
-			candidateNode := candidate.CreateChild(key.Value, value)
-			newMatches.Set(candidateNode.GetKey(), candidateNode)
+			if !prefs.DontIncludeMapValues {
+				log.Debug("including value")
+				candidateNode := candidate.CreateChild(key.Value, value)
+				newMatches.Set(candidateNode.GetKey(), candidateNode)
+			}
 		}
 	}
 

pkg/yqlib/operator_traverse_path_test.go

@@ -47,13 +47,30 @@ var traversePathOperatorScenarios = []expressionScenario{
 	},
 	{
 		description:    "Special characters",
-		subdescription: "Use quotes around path elements with special characters",
+		subdescription: "Use quotes with brackets around path elements with special characters",
 		document:       `{"{}": frog}`,
-		expression:     `."{}"`,
+		expression:     `.["{}"]`,
 		expected: []string{
 			"D0, P[{}], (!!str)::frog\n",
 		},
 	},
+	{
+		description:    "Keys with spaces",
+		subdescription: "Use quotes with brackets around path elements with special characters",
+		document:       `{"red rabbit": frog}`,
+		expression:     `.["red rabbit"]`,
+		expected: []string{
+			"D0, P[red rabbit], (!!str)::frog\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{"flying fox": frog}`,
+		expression: `.["flying fox"]`,
+		expected: []string{
+			"D0, P[flying fox], (!!str)::frog\n",
+		},
+	},
 	{
 		description:    "Dynamic keys",
 		subdescription: `Expressions within [] can be used to dynamically lookup / calculate keys`,
@@ -426,5 +443,5 @@ func TestTraversePathOperatorScenarios(t *testing.T) {
 	for _, tt := range traversePathOperatorScenarios {
 		testScenario(t, &tt)
 	}
-	documentScenarios(t, "Traverse", traversePathOperatorScenarios)
+	documentScenarios(t, "Traverse (Read)", traversePathOperatorScenarios)
 }

pkg/yqlib/operators.go

@@ -4,6 +4,7 @@ import (
 	"container/list"
 	"fmt"
 
+	"github.com/jinzhu/copier"
 	"gopkg.in/yaml.v3"
 )
 
@@ -88,16 +89,25 @@ func createBooleanCandidate(owner *CandidateNode, value bool) *CandidateNode {
 	return owner.CreateChild(nil, node)
 }
 
-func createTraversalTree(path []interface{}, traversePrefs traversePreferences) *ExpressionNode {
+func createTraversalTree(path []interface{}, traversePrefs traversePreferences, targetKey bool) *ExpressionNode {
 	if len(path) == 0 {
 		return &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
 	} else if len(path) == 1 {
-		return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Preferences: traversePrefs, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
+		lastPrefs := traversePrefs
+		if targetKey {
+			err := copier.Copy(&lastPrefs, traversePrefs)
+			if err != nil {
+				panic(err)
+			}
+			lastPrefs.IncludeMapKeys = true
+			lastPrefs.DontIncludeMapValues = true
+		}
+		return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Preferences: lastPrefs, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
 	}
 
 	return &ExpressionNode{
 		Operation: &Operation{OperationType: shortPipeOpType},
-		Lhs:       createTraversalTree(path[0:1], traversePrefs),
-		Rhs:       createTraversalTree(path[1:], traversePrefs),
+		Lhs:       createTraversalTree(path[0:1], traversePrefs, false),
+		Rhs:       createTraversalTree(path[1:], traversePrefs, targetKey),
 	}
 }

release_instructions.txt

@@ -2,11 +2,13 @@
 - increment version in snapcraft.yaml
 - increment version in github-action/Dockerfile
 - make sure local build passes
+- commit version update changes
 - tag git with same version number
 - commit vX tag - this will trigger github actions
 - use github actions to publish docker and make github release
 - check github updated yq action in marketplace
 
+
 - snapcraft
   - will auto create a candidate, test it works then promote
   

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.5.0'
+version: '4.6.1'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.