3.4.1

This is to patch a high security vulnerability.
Closes #550

Signed-off-by: Peter Benjamin <petermbenjamin@gmail.com>

Dockerfile

@@ -12,7 +12,7 @@ RUN CGO_ENABLED=0 make local build
 
 # Choose alpine as a base image to make this useful for CI, as many
 # CI tools expect an interactive shell inside the container
-FROM alpine:3.8 as production
+FROM alpine:3.12 as production
 
 COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq
 RUN chmod +x /usr/bin/yq

README.md

@@ -7,9 +7,6 @@ a lightweight and portable command-line YAML processor
 
 The aim of the project is to be the [jq](https://github.com/stedolan/jq) or sed of yaml files.
 
-## New version!
-V3 is officially out - if you've been using v2 and want/need to upgrade, checkout the [upgrade guide](https://mikefarah.gitbook.io/yq/upgrading-from-v2).
-
 ## Install
 
 ### [Download the latest binary](https://github.com/mikefarah/yq/releases/latest)
@@ -19,12 +16,6 @@ V3 is officially out - if you've been using v2 and want/need to upgrade, checkou
 brew install yq
 ```
 
-### Windows:
-```
-choco install yq
-```
-Supported by @chillum
-
 ### Ubuntu and other Linux distros supporting `snap` packages:
 ```
 snap install yq
@@ -48,29 +39,27 @@ sudo mv /etc/myfile.tmp /etc/myfile
 rm /etc/myfile.tmp
 ```
 
-### On Ubuntu 16.04 or higher from Debian package:
-```sh
-sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
-sudo add-apt-repository ppa:rmescandon/yq
-sudo apt update
-sudo apt install yq -y
-```
-Supported by @rmescandon
+### wget
 
-### Go Get:
-```
-GO111MODULE=on go get github.com/mikefarah/yq/v3
+Use wget to download the pre-compiled binaries:
+
+```bash
+wget https://github.com/mikefarah/yq/releases/download/{VERSION}/{BINARY} -O /usr/bin/yq &&\
+    chmod +x /usr/bin/yq
 ```
 
-## Run with Docker
+For instance, VERSION=3.4.0 and BINARY=yq_linux_amd64
 
-Oneshot use:
+
+### Run with Docker
+
+#### Oneshot use:
 
 ```bash
 docker run --rm -v "${PWD}":/workdir mikefarah/yq yq [flags] <command> FILE...
 ```
 
-Run commands interactively:
+#### Run commands interactively:
 
 ```bash
 docker run --rm -it -v "${PWD}":/workdir mikefarah/yq sh
@@ -84,6 +73,39 @@ yq() {
 }
 ```
 
+### Go Get:
+```
+GO111MODULE=on go get github.com/mikefarah/yq/v3
+```
+
+## Community Supported Installation methods
+As these are supported by the community :heart: - however, they may be out of date with the officially supported releases.
+
+
+### Windows:
+```
+choco install yq
+```
+Supported by @chillum (https://chocolatey.org/packages/yq)
+
+### Alpine Linux
+- Enable edge/community repo by adding ```$MIRROR/alpine/edge/community``` to ```/etc/apk/repositories```
+- Update database index with ```apk update```
+- Install yq with ```apk add yq```
+
+Supported by Tuan Hoang
+https://pkgs.alpinelinux.org/package/edge/community/x86/yq
+
+
+### On Ubuntu 16.04 or higher from Debian package:
+```sh
+sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
+sudo add-apt-repository ppa:rmescandon/yq
+sudo apt update
+sudo apt install yq -y
+```
+Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
+
 ## Features
 - Written in portable go, so you can download a lovely dependency free binary
 - [Colorize the output](https://mikefarah.gitbook.io/yq/usage/output-format#colorize-output)
@@ -136,5 +158,9 @@ Flags:
 Use "yq [command] --help" for more information about a command.
 ```
 
-## Known Issues
+## Upgrade from V2
+If you've been using v2 and want/need to upgrade, checkout the [upgrade guide](https://mikefarah.gitbook.io/yq/upgrading-from-v2).
+
+## Known Issues / Missing Features
 - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)
+- You cannot (yet) select multiple paths/keys from the yaml to be printed out (https://github.com/mikefarah/yq/issues/287)

cmd/constant.go

@@ -26,7 +26,8 @@ var defaultValue = ""
 var indent = 2
 var overwriteFlag = false
 var autoCreateFlag = true
-var appendFlag = false
+var arrayMergeStrategyFlag = "update"
+var commentsMergeStrategyFlag = "setWhenBlank"
 var verbose = false
 var version = false
 var docIndex = "0"

cmd/merge.go

@@ -11,14 +11,14 @@ func createMergeCmd() *cobra.Command {
 	var cmdMerge = &cobra.Command{
 		Use:     "merge [initial_yaml_file] [additional_yaml_file]...",
 		Aliases: []string{"m"},
-		Short:   "yq m [--inplace/-i] [--doc/-d index] [--overwrite/-x] [--append/-a] sample.yaml sample2.yaml",
+		Short:   "yq m [--inplace/-i] [--doc/-d index] [--overwrite/-x] [--arrayMerge/-a strategy] sample.yaml sample2.yaml",
 		Example: `
 yq merge things.yaml other.yaml
 yq merge --inplace things.yaml other.yaml
 yq m -i things.yaml other.yaml
 yq m --overwrite things.yaml other.yaml
 yq m -i -x things.yaml other.yaml
-yq m -i -a things.yaml other.yaml
+yq m -i -a=append things.yaml other.yaml
 yq m -i --autocreate=false things.yaml other.yaml
       `,
 		Long: `Updates the yaml file by adding/updating the path(s) and value(s) from additional yaml file(s).
@@ -32,7 +32,17 @@ If append flag is set then existing arrays will be merged with the arrays from e
 	cmdMerge.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace")
 	cmdMerge.PersistentFlags().BoolVarP(&overwriteFlag, "overwrite", "x", false, "update the yaml file by overwriting existing values")
 	cmdMerge.PersistentFlags().BoolVarP(&autoCreateFlag, "autocreate", "c", true, "automatically create any missing entries")
-	cmdMerge.PersistentFlags().BoolVarP(&appendFlag, "append", "a", false, "update the yaml file by appending array values")
+	cmdMerge.PersistentFlags().StringVarP(&arrayMergeStrategyFlag, "arrays", "a", "update", `array merge strategy (update/append/overwrite)
+update: recursively update arrays by their index
+append: concatenate arrays together
+overwrite: replace arrays
+`)
+	cmdMerge.PersistentFlags().StringVarP(&commentsMergeStrategyFlag, "comments", "", "setWhenBlank", `comments merge strategy (setWhenBlank/ignore/append/overwrite)
+setWhenBlank: set comment if the original document has no comment at that node
+ignore: leave comments as-is in the original
+append: append comments together
+overwrite: overwrite comments completely
+`)
 	cmdMerge.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
 	return cmdMerge
 }
@@ -41,9 +51,9 @@ If append flag is set then existing arrays will be merged with the arrays from e
 * We don't deeply traverse arrays when appending a merge, instead we want to
 * append the entire array element.
  */
-func createReadFunctionForMerge() func(*yaml.Node) ([]*yqlib.NodeContext, error) {
+func createReadFunctionForMerge(arrayMergeStrategy yqlib.ArrayMergeStrategy) func(*yaml.Node) ([]*yqlib.NodeContext, error) {
 	return func(dataBucket *yaml.Node) ([]*yqlib.NodeContext, error) {
-		return lib.Get(dataBucket, "**", !appendFlag)
+		return lib.GetForMerge(dataBucket, "**", arrayMergeStrategy)
 	}
 }
 
@@ -53,19 +63,59 @@ func mergeProperties(cmd *cobra.Command, args []string) error {
 	if len(args) < 1 {
 		return errors.New("Must provide at least 1 yaml file")
 	}
+	var arrayMergeStrategy yqlib.ArrayMergeStrategy
+
+	switch arrayMergeStrategyFlag {
+	case "update":
+		arrayMergeStrategy = yqlib.UpdateArrayMergeStrategy
+	case "append":
+		arrayMergeStrategy = yqlib.AppendArrayMergeStrategy
+	case "overwrite":
+		arrayMergeStrategy = yqlib.OverwriteArrayMergeStrategy
+	default:
+		return errors.New("Array merge strategy must be one of: update/append/overwrite")
+	}
+
+	var commentsMergeStrategy yqlib.CommentsMergeStrategy
+
+	switch commentsMergeStrategyFlag {
+	case "setWhenBlank":
+		commentsMergeStrategy = yqlib.SetWhenBlankCommentsMergeStrategy
+	case "ignore":
+		commentsMergeStrategy = yqlib.IgnoreCommentsMergeStrategy
+	case "append":
+		commentsMergeStrategy = yqlib.AppendCommentsMergeStrategy
+	case "overwrite":
+		commentsMergeStrategy = yqlib.OverwriteCommentsMergeStrategy
+	default:
+		return errors.New("Comments merge strategy must be one of: setWhenBlank/ignore/append/overwrite")
+	}
 
 	if len(args) > 1 {
 		// first generate update commands from the file
 		var filesToMerge = args[1:]
 
 		for _, fileToMerge := range filesToMerge {
-			matchingNodes, errorProcessingFile := doReadYamlFile(fileToMerge, createReadFunctionForMerge(), false, 0)
+			matchingNodes, errorProcessingFile := doReadYamlFile(fileToMerge, createReadFunctionForMerge(arrayMergeStrategy), false, 0)
 			if errorProcessingFile != nil {
 				return errorProcessingFile
 			}
+			log.Debugf("finished reading for merge!")
 			for _, matchingNode := range matchingNodes {
-				mergePath := lib.MergePathStackToString(matchingNode.PathStack, appendFlag)
-				updateCommands = append(updateCommands, yqlib.UpdateCommand{Command: "update", Path: mergePath, Value: matchingNode.Node, Overwrite: overwriteFlag})
+				log.Debugf("matched node %v", lib.PathStackToString(matchingNode.PathStack))
+				yqlib.DebugNode(matchingNode.Node)
+			}
+			for _, matchingNode := range matchingNodes {
+				mergePath := lib.MergePathStackToString(matchingNode.PathStack, arrayMergeStrategy)
+				updateCommands = append(updateCommands, yqlib.UpdateCommand{
+					Command:               "merge",
+					Path:                  mergePath,
+					Value:                 matchingNode.Node,
+					Overwrite:             overwriteFlag,
+					CommentsMergeStrategy: commentsMergeStrategy,
+					// dont update the content for nodes midway, only leaf nodes
+					DontUpdateNodeContent: matchingNode.IsMiddleNode && (arrayMergeStrategy != yqlib.OverwriteArrayMergeStrategy || matchingNode.Node.Kind != yaml.SequenceNode),
+				})
 			}
 		}
 	}

cmd/merge_test.go

@@ -60,7 +60,7 @@ func TestMergeOverwriteCmd(t *testing.T) {
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `a: other # better than the original
+	expectedOutput := `a: other # just the best
 b: [3, 4]
 c:
   test: 1
@@ -68,9 +68,36 @@ c:
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
+func TestMergeOverwriteDeepExampleCmd(t *testing.T) {
+	content := `c:
+  test: 1
+  thing: whatever
+`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	mergeContent := `c:
+  test: 5
+`
+	mergeFilename := test.WriteTempYamlFile(mergeContent)
+	defer test.RemoveTempYamlFile(mergeFilename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("merge --autocreate=false --overwrite %s %s", filename, mergeFilename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `c:
+  test: 5
+  thing: whatever
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestMergeAppendCmd(t *testing.T) {
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "merge --autocreate=false --append ../examples/data1.yaml ../examples/data2.yaml")
+	result := test.RunCmd(cmd, "merge --autocreate=false --arrays=append ../examples/data1.yaml ../examples/data2.yaml")
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
@@ -96,7 +123,7 @@ func TestMergeAppendArraysCmd(t *testing.T) {
 	defer test.RemoveTempYamlFile(mergeFilename)
 
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, fmt.Sprintf("merge --append -d* %s %s", filename, mergeFilename))
+	result := test.RunCmd(cmd, fmt.Sprintf("merge --arrays=append -d* %s %s", filename, mergeFilename))
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
@@ -109,13 +136,56 @@ func TestMergeAppendArraysCmd(t *testing.T) {
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
-func TestMergeOverwriteAndAppendCmd(t *testing.T) {
+func TestMergeAliasArraysCmd(t *testing.T) {
+	content := `
+vars:
+  variable1: &var1 cat
+
+usage:
+  value1: *var1
+  valueAnother: *var1
+  valuePlain: thing
+`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	mergeContent := `
+vars:
+  variable2: &var2 puppy
+
+usage:
+  value2: *var2
+  valueAnother: *var2
+  valuePlain: *var2
+`
+
+	mergeFilename := test.WriteTempYamlFile(mergeContent)
+	defer test.RemoveTempYamlFile(mergeFilename)
+
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "merge --autocreate=false --append --overwrite ../examples/data1.yaml ../examples/data2.yaml")
+	result := test.RunCmd(cmd, fmt.Sprintf("merge -x %s %s", filename, mergeFilename))
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `a: other # better than the original
+	expectedOutput := `vars:
+  variable1: &var1 cat
+  variable2: &var2 puppy
+usage:
+  value1: *var1
+  valueAnother: *var2
+  valuePlain: *var2
+  value2: *var2
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeOverwriteAndAppendCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "merge --autocreate=false --arrays=append --overwrite ../examples/data1.yaml ../examples/data2.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `a: other # just the best
 b: [1, 2, 3, 4]
 c:
   test: 1
@@ -123,13 +193,161 @@ c:
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
-func TestMergeArraysCmd(t *testing.T) {
+var commentContentA = `
+a: valueA1 # commentA1
+b: valueB1
+`
+
+var commentContentB = `
+a: valueA2 # commentA2
+b: valueB2 # commentB2
+c: valueC2 # commentC2
+`
+
+func TestMergeCommentsSetWhenBlankCmd(t *testing.T) {
+	filename := test.WriteTempYamlFile(commentContentA)
+	defer test.RemoveTempYamlFile(filename)
+
+	mergeFilename := test.WriteTempYamlFile(commentContentB)
+	defer test.RemoveTempYamlFile(mergeFilename)
+
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "merge --append ../examples/sample_array.yaml ../examples/sample_array_2.yaml")
+	result := test.RunCmd(cmd, fmt.Sprintf("merge --comments=setWhenBlank %s %s", filename, mergeFilename))
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `[1, 2, 3, 4, 5]
+
+	expectedOutput := `a: valueA1 # commentA1
+b: valueB1 # commentB2
+c: valueC2 # commentC2
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeCommentsIgnoreCmd(t *testing.T) {
+	filename := test.WriteTempYamlFile(commentContentA)
+	defer test.RemoveTempYamlFile(filename)
+
+	mergeFilename := test.WriteTempYamlFile(commentContentB)
+	defer test.RemoveTempYamlFile(mergeFilename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("merge --comments=ignore %s %s", filename, mergeFilename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `a: valueA1 # commentA1
+b: valueB1
+c: valueC2
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeCommentsAppendCmd(t *testing.T) {
+	filename := test.WriteTempYamlFile(commentContentA)
+	defer test.RemoveTempYamlFile(filename)
+
+	mergeFilename := test.WriteTempYamlFile(commentContentB)
+	defer test.RemoveTempYamlFile(mergeFilename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("merge --comments=append %s %s", filename, mergeFilename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `a: valueA1 # commentA1 # commentA2
+b: valueB1 # commentB2
+c: valueC2 # commentC2
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeCommentsOverwriteCmd(t *testing.T) {
+	filename := test.WriteTempYamlFile(commentContentA)
+	defer test.RemoveTempYamlFile(filename)
+
+	mergeFilename := test.WriteTempYamlFile(commentContentB)
+	defer test.RemoveTempYamlFile(mergeFilename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("merge --comments=overwrite %s %s", filename, mergeFilename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `a: valueA1 # commentA2
+b: valueB1 # commentB2
+c: valueC2 # commentC2
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeOverwriteArraysTooCmd(t *testing.T) {
+	content := `a: simple # just the best
+b: [1, 2]
+c:
+  test: 1
+`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	mergeContent := `a: things
+b: [6]`
+	mergeFilename := test.WriteTempYamlFile(mergeContent)
+	defer test.RemoveTempYamlFile(mergeFilename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("merge --autocreate=false --arrays=overwrite --overwrite %s %s", filename, mergeFilename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `a: things # just the best
+b: [6]
+c:
+  test: 1
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeRootArraysCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "merge --arrays=append ../examples/sample_array.yaml ../examples/sample_array_2.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `- 1
+- 2
+- 3
+- 4
+- 5
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeOverwriteArraysCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "merge --arrays=overwrite ../examples/sample_array.yaml ../examples/sample_array_2.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `- 4
+- 5
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeUpdateArraysCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "merge -x --arrays=update ../examples/sample_array.yaml ../examples/sample_array_2.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `- 4
+- 5
+- 3
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -145,9 +363,7 @@ func TestMergeCmd_Multi(t *testing.T) {
 another:
   document: here
 a: simple # just the best
-b:
-  - 1
-  - 2
+b: [1, 2]
 c:
   test: 1
 ---
@@ -316,9 +532,7 @@ func TestMergeAllowEmptyTargetCmd(t *testing.T) {
 		t.Error(result.Error)
 	}
 	expectedOutput := `a: simple # just the best
-b:
-  - 1
-  - 2
+b: [1, 2]
 c:
   test: 1
 `

cmd/new.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"github.com/mikefarah/yq/v3/pkg/yqlib"
-	errors "github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
 
@@ -36,11 +35,7 @@ Note that you can give a create script to perform more sophisticated yaml. This
 
 func newProperty(cmd *cobra.Command, args []string) error {
 	var badArgsMessage = "Must provide <path_to_update> <value>"
-	if len(args) != 2 {
-		return errors.New(badArgsMessage)
-	}
-
-	var updateCommands, updateCommandsError = readUpdateCommands(args, 2, badArgsMessage)
+	var updateCommands, updateCommandsError = readUpdateCommands(args, 2, badArgsMessage, false)
 	if updateCommandsError != nil {
 		return updateCommandsError
 	}

cmd/new_test.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/mikefarah/yq/v3/test"
@@ -18,6 +19,24 @@ func TestNewCmd(t *testing.T) {
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
+func TestNewCmdScript(t *testing.T) {
+	updateScript := `- command: update
+  path: b.c
+  value: 7`
+	scriptFilename := test.WriteTempYamlFile(updateScript)
+	defer test.RemoveTempYamlFile(scriptFilename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("new --script %s", scriptFilename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `b:
+  c: 7
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestNewAnchorCmd(t *testing.T) {
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, "new b.c 3 --anchorName=fred")

cmd/read_test.go

@@ -94,6 +94,28 @@ func TestReadUnwrapJsonByDefaultCmd(t *testing.T) {
 	test.AssertResult(t, "\"frog\"\n", result.Output)
 }
 
+func TestReadOutputJsonNonStringKeysCmd(t *testing.T) {
+
+	content := `
+true: true
+5:
+  null:
+    0.1: deeply
+    false: things`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("read %s -j", filename))
+
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `{"true":true,"5":{"null":{"0.1":"deeply","false":"things"}}}
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestReadWithAdvancedFilterCmd(t *testing.T) {
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, "read ../examples/sample.yaml b.e(name==sam).value")
@@ -192,7 +214,7 @@ func TestReadArrayLengthCmd(t *testing.T) {
 }
 
 func TestReadArrayLengthDeepCmd(t *testing.T) {
-	content := `holder: 
+	content := `holder:
 - things
 - whatever
 `
@@ -208,12 +230,12 @@ func TestReadArrayLengthDeepCmd(t *testing.T) {
 }
 
 func TestReadArrayLengthDeepMultipleCmd(t *testing.T) {
-	content := `holderA: 
+	content := `holderA:
 - things
 - whatever
 skipMe:
 - yep
-holderB: 
+holderB:
 - other things
 - cool
 `
@@ -272,10 +294,10 @@ func TestReadCollectArrayCmd(t *testing.T) {
 }
 
 func TestReadArrayLengthDeepMultipleWithPathCmd(t *testing.T) {
-	content := `holderA: 
+	content := `holderA:
 - things
 - whatever
-holderB: 
+holderB:
 - other things
 - cool
 `
@@ -306,7 +328,7 @@ dog: bark
 }
 
 func TestReadObjectLengthDeepCmd(t *testing.T) {
-	content := `holder: 
+	content := `holder:
   cat: meow
   dog: bark
 `
@@ -322,10 +344,10 @@ func TestReadObjectLengthDeepCmd(t *testing.T) {
 }
 
 func TestReadObjectLengthDeepMultipleCmd(t *testing.T) {
-	content := `holderA: 
+	content := `holderA:
   cat: meow
   dog: bark
-holderB: 
+holderB:
   elephant: meow
   zebra: bark
 `
@@ -341,10 +363,10 @@ holderB:
 }
 
 func TestReadObjectLengthDeepMultipleWithPathsCmd(t *testing.T) {
-	content := `holderA: 
+	content := `holderA:
   cat: meow
   dog: bark
-holderB: 
+holderB:
   elephant: meow
   zebra: bark
 `
@@ -400,7 +422,7 @@ func TestReadSingleQuotedStringCmd(t *testing.T) {
 
 func TestReadQuotedMultinlineStringCmd(t *testing.T) {
 	content := `test: |
-  abcdefg    
+  abcdefg
   hijklmno
 `
 	filename := test.WriteTempYamlFile(content)
@@ -411,7 +433,7 @@ func TestReadQuotedMultinlineStringCmd(t *testing.T) {
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `abcdefg    
+	expectedOutput := `abcdefg
 hijklmno
 
 `
@@ -420,7 +442,7 @@ hijklmno
 
 func TestReadQuotedMultinlineNoNewLineStringCmd(t *testing.T) {
 	content := `test: |-
-  abcdefg    
+  abcdefg
   hijklmno
 `
 	filename := test.WriteTempYamlFile(content)
@@ -431,7 +453,7 @@ func TestReadQuotedMultinlineNoNewLineStringCmd(t *testing.T) {
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `abcdefg    
+	expectedOutput := `abcdefg
 hijklmno
 `
 	test.AssertResult(t, expectedOutput, result.Output)
@@ -561,7 +583,7 @@ func TestReadMergeAnchorsExplodeJsonCmd(t *testing.T) {
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `{"bar":{"b":2,"c":"oldbar","thing":"coconut"},"foo":{"a":"original","thing":"coolasdf","thirsty":"yep"},"foobar":{"a":"original","c":3,"thing":"ice","thirsty":"yep","thirty":"well beyond"},"foobarList":{"a":"original","b":2,"c":"newbar","thing":"coconut","thirsty":"yep"}}
+	expectedOutput := `{"foo":{"a":"original","thing":"coolasdf","thirsty":"yep"},"bar":{"b":2,"thing":"coconut","c":"oldbar"},"foobarList":{"c":"newbar","b":2,"thing":"coconut","a":"original","thirsty":"yep"},"foobar":{"thirty":"well beyond","thing":"ice","c":3,"a":"original","thirsty":"yep"}}
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -595,6 +617,44 @@ pointer: *value-pointer`
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
+func TestReadMergeAnchorsExplodeMissingCmd(t *testing.T) {
+	content := `a:
+  <<: &anchor
+  c: d
+  e: f
+`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("read -X %s", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `a:
+  c: d
+  e: f
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestReadMergeAnchorsExplodeKeyCmd(t *testing.T) {
+	content := `name: &nameField Mike
+*nameField: Great Guy`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("read -X %s", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `name: Mike
+Mike: Great Guy
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestReadMergeAnchorsExplodeSimpleArrayCmd(t *testing.T) {
 	content := `- things`
 	filename := test.WriteTempYamlFile(content)
@@ -850,9 +910,9 @@ func TestReadEmptyContentCmd(t *testing.T) {
 }
 
 func TestReadEmptyNodesPrintPathCmd(t *testing.T) {
-	content := `map: 
+	content := `map:
   that: {}
-array: 
+array:
   great: []
 null:
   indeed: ~`
@@ -902,6 +962,7 @@ b:
       value: 3
     - name: sam
       value: 4
+ab: must appear last
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -941,6 +1002,7 @@ b:
           value: 3
         - name: sam
           value: 4
+ab: must appear last
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -1209,7 +1271,7 @@ func TestReadBadDataCmd(t *testing.T) {
 
 func TestReadDeepFromRootCmd(t *testing.T) {
 	content := `state:
-  country: 
+  country:
     city: foo
 `
 	filename := test.WriteTempYamlFile(content)
@@ -1379,3 +1441,23 @@ func TestReadFindValueDeepObjectCmd(t *testing.T) {
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
+
+func TestReadKeepsKeyOrderInJson(t *testing.T) {
+	const content = `{
+	"z": "One",
+	"a": 1,
+	"w": ["a", "r"],
+	"u": {"d": "o", "0": 11.5},
+}`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("r -j %s", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `{"z":"One","a":1,"w":["a","r"],"u":{"d":"o","0":11.5}}
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}

cmd/root.go

@@ -22,6 +22,7 @@ func New() *cobra.Command {
 			return nil
 		},
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			cmd.SetOut(cmd.OutOrStdout())
 			var format = logging.MustStringFormatter(
 				`%{color}%{time:15:04:05} %{shortfunc} [%{level:.4s}]%{color:reset} %{message}`,
 			)

cmd/utils.go

@@ -17,7 +17,7 @@ type readDataFn func(dataBucket *yaml.Node) ([]*yqlib.NodeContext, error)
 
 func createReadFunction(path string) func(*yaml.Node) ([]*yqlib.NodeContext, error) {
 	return func(dataBucket *yaml.Node) ([]*yqlib.NodeContext, error) {
-		return lib.Get(dataBucket, path, true)
+		return lib.Get(dataBucket, path)
 	}
 }
 
@@ -163,6 +163,9 @@ func setIfNotThere(node *yaml.Node, key string, value *yaml.Node) {
 }
 
 func applyAlias(node *yaml.Node, alias *yaml.Node) {
+	if alias == nil {
+		return
+	}
 	for index := 0; index < len(alias.Content); index = index + 2 {
 		keyNode := alias.Content[index]
 		log.Debugf("applying alias key %v", keyNode.Value)
@@ -185,12 +188,14 @@ func explodeNode(node *yaml.Node) error {
 		return nil
 	case yaml.AliasNode:
 		log.Debugf("its an alias!")
-		node.Kind = node.Alias.Kind
-		node.Style = node.Alias.Style
-		node.Tag = node.Alias.Tag
-		node.Content = node.Alias.Content
-		node.Value = node.Alias.Value
-		node.Alias = nil
+		if node.Alias != nil {
+			node.Kind = node.Alias.Kind
+			node.Style = node.Alias.Style
+			node.Tag = node.Alias.Tag
+			node.Content = node.Alias.Content
+			node.Value = node.Alias.Value
+			node.Alias = nil
+		}
 		return nil
 	case yaml.MappingNode:
 		for index := 0; index < len(node.Content); index = index + 2 {
@@ -202,6 +207,10 @@ func explodeNode(node *yaml.Node) error {
 				if errorInContent != nil {
 					return errorInContent
 				}
+				errorInContent = explodeNode(keyNode)
+				if errorInContent != nil {
+					return errorInContent
+				}
 			} else {
 				if valueNode.Kind == yaml.SequenceNode {
 					log.Debugf("an alias merge list!")
@@ -477,7 +486,7 @@ type updateCommandParsed struct {
 	Value   yaml.Node
 }
 
-func readUpdateCommands(args []string, expectedArgs int, badArgsMessage string) ([]yqlib.UpdateCommand, error) {
+func readUpdateCommands(args []string, expectedArgs int, badArgsMessage string, allowNoValue bool) ([]yqlib.UpdateCommand, error) {
 	var updateCommands []yqlib.UpdateCommand = make([]yqlib.UpdateCommand, 0)
 	if writeScript != "" {
 		var parsedCommands = make([]updateCommandParsed, 0)
@@ -512,8 +521,8 @@ func readUpdateCommands(args []string, expectedArgs int, badArgsMessage string)
 		log.Debug("path %v", args[expectedArgs-2])
 		log.Debug("Value %v", args[expectedArgs-1])
 		value := valueParser.Parse(args[expectedArgs-1], customTag, customStyle, anchorName, makeAlias)
-		updateCommands[0] = yqlib.UpdateCommand{Command: "update", Path: args[expectedArgs-2], Value: value, Overwrite: true, DontUpdateComments: true}
-	} else if len(args) == expectedArgs-1 {
+		updateCommands[0] = yqlib.UpdateCommand{Command: "update", Path: args[expectedArgs-2], Value: value, Overwrite: true, CommentsMergeStrategy: yqlib.IgnoreCommentsMergeStrategy}
+	} else if len(args) == expectedArgs-1 && allowNoValue {
 		// don't update the value
 		updateCommands = make([]yqlib.UpdateCommand, 1)
 		log.Debug("args %v", args)

cmd/validate.go

@@ -15,7 +15,7 @@ yq v - # reads from stdin
 `,
 		RunE:          validateProperty,
 		SilenceUsage:  true,
-		SilenceErrors: true,
+		SilenceErrors: false,
 	}
 	cmdRead.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
 	return cmdRead

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "3.3.2"
+	Version = "3.4.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

cmd/write.go

@@ -53,7 +53,7 @@ format is list of update commands (update or delete) like so:
 }
 
 func writeProperty(cmd *cobra.Command, args []string) error {
-	var updateCommands, updateCommandsError = readUpdateCommands(args, 3, "Must provide <filename> <path_to_update> <value>")
+	var updateCommands, updateCommandsError = readUpdateCommands(args, 3, "Must provide <filename> <path_to_update> <value>", true)
 	if updateCommandsError != nil {
 		return updateCommandsError
 	}

examples/sample.json

@@ -1 +1 @@
-{"a":"Easy! as one two three","b":{"c":2,"d":[3,4],"e":[{"name":"fred","value":3},{"name":"sam","value":4}]}}
+{"a":"Easy! as one two three","b":{"c":2,"d":[3,4],"e":[{"name":"fred","value":3},{"name":"sam","value":4}]},"ab":"must appear last"}

go.sum

@@ -152,6 +152,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

pkg/yqlib/data_navigator.go

@@ -22,13 +22,8 @@ func NewDataNavigator(NavigationStrategy NavigationStrategy) DataNavigator {
 }
 
 func (n *navigator) Traverse(value *yaml.Node, path []interface{}) error {
-	realValue := value
 	emptyArray := make([]interface{}, 0)
 	log.Debugf("Traversing path %v", pathStackToString(path))
-	if realValue.Kind == yaml.DocumentNode {
-		log.Debugf("its a document! returning the first child")
-		return n.doTraverse(value.Content[0], "", path, emptyArray)
-	}
 	return n.doTraverse(value, "", path, emptyArray)
 }
 
@@ -39,7 +34,8 @@ func (n *navigator) doTraverse(value *yaml.Node, head interface{}, tail []interf
 	var nodeContext = NewNodeContext(value, head, tail, pathStack)
 
 	var errorDeepSplatting error
-	if head == "**" && value.Kind != yaml.ScalarNode && n.navigationStrategy.ShouldDeeplyTraverse(nodeContext) {
+	// no need to deeply traverse the DocumentNode, as it's already covered by its first child.
+	if head == "**" && value.Kind != yaml.DocumentNode && value.Kind != yaml.ScalarNode && n.navigationStrategy.ShouldDeeplyTraverse(nodeContext) {
 		if len(pathStack) == 0 || pathStack[len(pathStack)-1] != "<<" {
 			errorDeepSplatting = n.recurse(value, head, tail, pathStack)
 		}
@@ -51,7 +47,11 @@ func (n *navigator) doTraverse(value *yaml.Node, head interface{}, tail []interf
 		return errorDeepSplatting
 	}
 
-	if len(tail) > 0 && value.Kind != yaml.ScalarNode {
+	if value.Kind == yaml.DocumentNode {
+		log.Debugf("its a document, diving into %v", head)
+		DebugNode(value)
+		return n.recurse(value, head, tail, pathStack)
+	} else if len(tail) > 0 && value.Kind != yaml.ScalarNode {
 		log.Debugf("diving into %v", tail[0])
 		DebugNode(value)
 		return n.recurse(value, tail[0], tail[1:], pathStack)
@@ -73,6 +73,7 @@ func (n *navigator) recurse(value *yaml.Node, head interface{}, tail []interface
 	nodeContext := NewNodeContext(value, head, tail, pathStack)
 
 	if head == "**" && !n.navigationStrategy.ShouldOnlyDeeplyVisitLeaves(nodeContext) {
+		nodeContext.IsMiddleNode = true
 		errorVisitingDeeply := n.navigationStrategy.Visit(nodeContext)
 		if errorVisitingDeeply != nil {
 			return errorVisitingDeeply
@@ -108,6 +109,8 @@ func (n *navigator) recurse(value *yaml.Node, head interface{}, tail []interface
 			return n.recurse(value.Alias, head, tail, pathStack)
 		}
 		return nil
+	case yaml.DocumentNode:
+		return n.doTraverse(value.Content[0], head, tail, pathStack)
 	default:
 		return n.navigationStrategy.Visit(nodeContext)
 	}

pkg/yqlib/encoder.go

@@ -3,6 +3,7 @@ package yqlib
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"io"
 
 	yaml "gopkg.in/yaml.v3"
@@ -58,6 +59,21 @@ type jsonEncoder struct {
 	encoder *json.Encoder
 }
 
+func mapKeysToStrings(node *yaml.Node) {
+
+	if node.Kind == yaml.MappingNode {
+		for index, child := range node.Content {
+			if index%2 == 0 { // its a map key
+				child.Tag = "!!str"
+			}
+		}
+	}
+
+	for _, child := range node.Content {
+		mapKeysToStrings(child)
+	}
+}
+
 func NewJsonEncoder(destination io.Writer, prettyPrint bool, indent int) Encoder {
 	var encoder = json.NewEncoder(destination)
 	var indentString = ""
@@ -72,10 +88,162 @@ func NewJsonEncoder(destination io.Writer, prettyPrint bool, indent int) Encoder
 }
 
 func (je *jsonEncoder) Encode(node *yaml.Node) error {
-	var dataBucket interface{}
+	var dataBucket orderedMap
+	// firstly, convert all map keys to strings
+	mapKeysToStrings(node)
 	errorDecoding := node.Decode(&dataBucket)
 	if errorDecoding != nil {
 		return errorDecoding
 	}
 	return je.encoder.Encode(dataBucket)
 }
+
+// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the
+// order of keys and values in a map or an object.
+type orderedMap struct {
+	// if this is an object, kv != nil. If this is not an object, kv == nil.
+	kv     []orderedMapKV
+	altVal interface{}
+}
+
+type orderedMapKV struct {
+	K string
+	V orderedMap
+}
+
+func (o *orderedMap) UnmarshalJSON(data []byte) error {
+	switch data[0] {
+	case '{':
+		// initialise so that even if the object is empty it is not nil
+		o.kv = []orderedMapKV{}
+
+		// create decoder
+		dec := json.NewDecoder(bytes.NewReader(data))
+		_, err := dec.Token() // open object
+		if err != nil {
+			return err
+		}
+
+		// cycle through k/v
+		var tok json.Token
+		for tok, err = dec.Token(); err != io.EOF; tok, err = dec.Token() {
+			// we can expect two types: string or Delim. Delim automatically means
+			// that it is the closing bracket of the object, whereas string means
+			// that there is another key.
+			if _, ok := tok.(json.Delim); ok {
+				break
+			}
+			kv := orderedMapKV{
+				K: tok.(string),
+			}
+			if err := dec.Decode(&kv.V); err != nil {
+				return err
+			}
+			o.kv = append(o.kv, kv)
+		}
+		// unexpected error
+		if err != nil && err != io.EOF {
+			return err
+		}
+		return nil
+	case '[':
+		var arr []orderedMap
+		return json.Unmarshal(data, &arr)
+	}
+
+	return json.Unmarshal(data, &o.altVal)
+}
+
+func (o orderedMap) MarshalJSON() ([]byte, error) {
+	if o.kv == nil {
+		return json.Marshal(o.altVal)
+	}
+	buf := new(bytes.Buffer)
+	enc := json.NewEncoder(buf)
+	buf.WriteByte('{')
+	for idx, el := range o.kv {
+		if err := enc.Encode(el.K); err != nil {
+			return nil, err
+		}
+		buf.WriteByte(':')
+		if err := enc.Encode(el.V); err != nil {
+			return nil, err
+		}
+		if idx != len(o.kv)-1 {
+			buf.WriteByte(',')
+		}
+	}
+	buf.WriteByte('}')
+	return buf.Bytes(), nil
+}
+
+func (o *orderedMap) UnmarshalYAML(node *yaml.Node) error {
+	switch node.Kind {
+	case yaml.DocumentNode:
+		if len(node.Content) == 0 {
+			return nil
+		}
+		return o.UnmarshalYAML(node.Content[0])
+	case yaml.AliasNode:
+		return o.UnmarshalYAML(node.Alias)
+	case yaml.ScalarNode:
+		return node.Decode(&o.altVal)
+	case yaml.MappingNode:
+		// set kv to non-nil
+		o.kv = []orderedMapKV{}
+		for i := 0; i < len(node.Content); i += 2 {
+			var key string
+			var val orderedMap
+			if err := node.Content[i].Decode(&key); err != nil {
+				return err
+			}
+			if err := node.Content[i+1].Decode(&val); err != nil {
+				return err
+			}
+			o.kv = append(o.kv, orderedMapKV{
+				K: key,
+				V: val,
+			})
+		}
+		return nil
+	case yaml.SequenceNode:
+		var res []orderedMap
+		if err := node.Decode(&res); err != nil {
+			return err
+		}
+		o.altVal = res
+		o.kv = nil
+		return nil
+	case 0:
+		// null
+		o.kv = nil
+		o.altVal = nil
+		return nil
+	default:
+		return fmt.Errorf("orderedMap: invalid yaml node")
+	}
+}
+
+func (o *orderedMap) MarshalYAML() (interface{}, error) {
+	// fast path: kv is nil, use altVal
+	if o.kv == nil {
+		return o.altVal, nil
+	}
+	content := make([]*yaml.Node, 0, len(o.kv)*2)
+	for _, val := range o.kv {
+		n := new(yaml.Node)
+		if err := n.Encode(val.V); err != nil {
+			return nil, err
+		}
+		content = append(content, &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!str",
+			Value: val.K,
+		}, n)
+	}
+	return &yaml.Node{
+		Kind:    yaml.MappingNode,
+		Tag:     "!!map",
+		Content: content,
+	}, nil
+}

pkg/yqlib/lib.go

@@ -13,12 +13,13 @@ import (
 var log = logging.MustGetLogger("yq")
 
 type UpdateCommand struct {
-	Command             string
-	Path                string
-	Value               *yaml.Node
-	Overwrite           bool
-	DontUpdateNodeValue bool
-	DontUpdateComments  bool
+	Command               string
+	Path                  string
+	Value                 *yaml.Node
+	Overwrite             bool
+	DontUpdateNodeValue   bool
+	DontUpdateNodeContent bool
+	CommentsMergeStrategy CommentsMergeStrategy
 }
 
 func KindString(kind yaml.Kind) string {
@@ -50,20 +51,23 @@ func DebugNode(value *yaml.Node) {
 		}
 		encoder.Close()
 		log.Debug("Tag: %v, Kind: %v, Anchor: %v", value.Tag, KindString(value.Kind), value.Anchor)
-		log.Debug("%v", buf.String())
+		log.Debug("Head Comment: %v", value.HeadComment)
+		log.Debug("Line Comment: %v", value.LineComment)
+		log.Debug("FootComment Comment: %v", value.FootComment)
+		log.Debug("\n%v", buf.String())
 	}
 }
 
 func pathStackToString(pathStack []interface{}) string {
-	return mergePathStackToString(pathStack, false)
+	return mergePathStackToString(pathStack, UpdateArrayMergeStrategy)
 }
 
-func mergePathStackToString(pathStack []interface{}, appendArrays bool) string {
+func mergePathStackToString(pathStack []interface{}, arrayMergeStrategy ArrayMergeStrategy) string {
 	var sb strings.Builder
 	for index, path := range pathStack {
 		switch path.(type) {
 		case int, int64:
-			if appendArrays {
+			if arrayMergeStrategy == AppendArrayMergeStrategy {
 				sb.WriteString("[+]")
 			} else {
 				sb.WriteString(fmt.Sprintf("[%v]", path))
@@ -94,9 +98,7 @@ func mergePathStackToString(pathStack []interface{}, appendArrays bool) string {
 			sb.WriteString(".")
 		}
 	}
-	var pathString = sb.String()
-	log.Debug("got a path string: %v", pathString)
-	return pathString
+	return sb.String()
 }
 
 func guessKind(head interface{}, tail []interface{}, guess yaml.Kind) yaml.Kind {
@@ -134,12 +136,13 @@ func guessKind(head interface{}, tail []interface{}, guess yaml.Kind) yaml.Kind
 }
 
 type YqLib interface {
-	Get(rootNode *yaml.Node, path string, deeplyTraverseArrays bool) ([]*NodeContext, error)
+	Get(rootNode *yaml.Node, path string) ([]*NodeContext, error)
+	GetForMerge(rootNode *yaml.Node, path string, arrayMergeStrategy ArrayMergeStrategy) ([]*NodeContext, error)
 	Update(rootNode *yaml.Node, updateCommand UpdateCommand, autoCreate bool) error
 	New(path string) yaml.Node
 
 	PathStackToString(pathStack []interface{}) string
-	MergePathStackToString(pathStack []interface{}, appendArrays bool) string
+	MergePathStackToString(pathStack []interface{}, arrayMergeStrategy ArrayMergeStrategy) string
 }
 
 type lib struct {
@@ -152,21 +155,28 @@ func NewYqLib() YqLib {
 	}
 }
 
-func (l *lib) Get(rootNode *yaml.Node, path string, deeplyTraverseArrays bool) ([]*NodeContext, error) {
+func (l *lib) Get(rootNode *yaml.Node, path string) ([]*NodeContext, error) {
 	var paths = l.parser.ParsePath(path)
-	navigationStrategy := ReadNavigationStrategy(deeplyTraverseArrays)
+	navigationStrategy := ReadNavigationStrategy()
 	navigator := NewDataNavigator(navigationStrategy)
 	error := navigator.Traverse(rootNode, paths)
 	return navigationStrategy.GetVisitedNodes(), error
+}
 
+func (l *lib) GetForMerge(rootNode *yaml.Node, path string, arrayMergeStrategy ArrayMergeStrategy) ([]*NodeContext, error) {
+	var paths = l.parser.ParsePath(path)
+	navigationStrategy := ReadForMergeNavigationStrategy(arrayMergeStrategy)
+	navigator := NewDataNavigator(navigationStrategy)
+	error := navigator.Traverse(rootNode, paths)
+	return navigationStrategy.GetVisitedNodes(), error
 }
 
 func (l *lib) PathStackToString(pathStack []interface{}) string {
 	return pathStackToString(pathStack)
 }
 
-func (l *lib) MergePathStackToString(pathStack []interface{}, appendArrays bool) string {
-	return mergePathStackToString(pathStack, appendArrays)
+func (l *lib) MergePathStackToString(pathStack []interface{}, arrayMergeStrategy ArrayMergeStrategy) string {
+	return mergePathStackToString(pathStack, arrayMergeStrategy)
 }
 
 func (l *lib) New(path string) yaml.Node {
@@ -182,6 +192,10 @@ func (l *lib) Update(rootNode *yaml.Node, updateCommand UpdateCommand, autoCreat
 		var paths = l.parser.ParsePath(updateCommand.Path)
 		navigator := NewDataNavigator(UpdateNavigationStrategy(updateCommand, autoCreate))
 		return navigator.Traverse(rootNode, paths)
+	case "merge":
+		var paths = l.parser.ParsePath(updateCommand.Path)
+		navigator := NewDataNavigator(MergeNavigationStrategy(updateCommand, autoCreate))
+		return navigator.Traverse(rootNode, paths)
 	case "delete":
 		var paths = l.parser.ParsePath(updateCommand.Path)
 		lastBit, newTail := paths[len(paths)-1], paths[:len(paths)-1]

pkg/yqlib/lib_test.go

@@ -30,7 +30,7 @@ func TestLib(t *testing.T) {
 		array[0] = "a"
 		array[1] = 0
 		array[2] = "b"
-		got := subject.MergePathStackToString(array, true)
+		got := subject.MergePathStackToString(array, AppendArrayMergeStrategy)
 		test.AssertResult(t, `a.[+].b`, got)
 	})
 

pkg/yqlib/merge_navigation_strategy.go

@@ -0,0 +1,103 @@
+package yqlib
+
+import "gopkg.in/yaml.v3"
+
+type ArrayMergeStrategy uint32
+
+const (
+	UpdateArrayMergeStrategy ArrayMergeStrategy = 1 << iota
+	OverwriteArrayMergeStrategy
+	AppendArrayMergeStrategy
+)
+
+type CommentsMergeStrategy uint32
+
+const (
+	SetWhenBlankCommentsMergeStrategy CommentsMergeStrategy = 1 << iota
+	IgnoreCommentsMergeStrategy
+	OverwriteCommentsMergeStrategy
+	AppendCommentsMergeStrategy
+)
+
+func MergeNavigationStrategy(updateCommand UpdateCommand, autoCreate bool) NavigationStrategy {
+	return &NavigationStrategyImpl{
+		visitedNodes: []*NodeContext{},
+		pathParser:   NewPathParser(),
+		followAlias: func(nodeContext NodeContext) bool {
+			return false
+		},
+		autoCreateMap: func(nodeContext NodeContext) bool {
+			return autoCreate
+		},
+		visit: func(nodeContext NodeContext) error {
+			node := nodeContext.Node
+			changesToApply := updateCommand.Value
+
+			if node.Kind == yaml.DocumentNode && changesToApply.Kind != yaml.DocumentNode {
+				// when the path is empty, it matches both the top level pseudo document node
+				// and the actual top level node (e.g. map/sequence/whatever)
+				// so when we are updating with no path, make sure we update the right node.
+				node = node.Content[0]
+			}
+
+			log.Debug("going to update")
+			DebugNode(node)
+			log.Debug("with")
+			DebugNode(changesToApply)
+
+			if updateCommand.Overwrite || node.Value == "" {
+				node.Value = changesToApply.Value
+				node.Tag = changesToApply.Tag
+				node.Kind = changesToApply.Kind
+				node.Style = changesToApply.Style
+				node.Anchor = changesToApply.Anchor
+				node.Alias = changesToApply.Alias
+
+				if !updateCommand.DontUpdateNodeContent {
+					node.Content = changesToApply.Content
+				}
+			} else {
+				log.Debug("skipping update as node already has value %v and overwriteFlag is ", node.Value, updateCommand.Overwrite)
+			}
+
+			switch updateCommand.CommentsMergeStrategy {
+			case OverwriteCommentsMergeStrategy:
+				node.HeadComment = changesToApply.HeadComment
+				node.LineComment = changesToApply.LineComment
+				node.FootComment = changesToApply.FootComment
+			case SetWhenBlankCommentsMergeStrategy:
+				if node.HeadComment == "" {
+					node.HeadComment = changesToApply.HeadComment
+				}
+				if node.LineComment == "" {
+					node.LineComment = changesToApply.LineComment
+				}
+				if node.FootComment == "" {
+					node.FootComment = changesToApply.FootComment
+				}
+			case AppendCommentsMergeStrategy:
+				if node.HeadComment == "" {
+					node.HeadComment = changesToApply.HeadComment
+				} else {
+					node.HeadComment = node.HeadComment + "\n" + changesToApply.HeadComment
+				}
+				if node.LineComment == "" {
+					node.LineComment = changesToApply.LineComment
+				} else {
+					node.LineComment = node.LineComment + " " + changesToApply.LineComment
+				}
+				if node.FootComment == "" {
+					node.FootComment = changesToApply.FootComment
+				} else {
+					node.FootComment = node.FootComment + "\n" + changesToApply.FootComment
+				}
+			default:
+			}
+
+			log.Debug("result")
+			DebugNode(node)
+
+			return nil
+		},
+	}
+}

pkg/yqlib/navigation_strategy.go

@@ -11,6 +11,9 @@ type NodeContext struct {
 	Head      interface{}
 	Tail      []interface{}
 	PathStack []interface{}
+	// middle nodes are nodes that match along the original path, but not a
+	// target match of the path. This is only relevant when ShouldOnlyDeeplyVisitLeaves is false.
+	IsMiddleNode bool
 }
 
 func NewNodeContext(node *yaml.Node, head interface{}, tail []interface{}, pathStack []interface{}) NodeContext {

pkg/yqlib/read_for_merge_navigation_strategy.go

@@ -0,0 +1,37 @@
+package yqlib
+
+import "gopkg.in/yaml.v3"
+
+func ReadForMergeNavigationStrategy(arrayMergeStrategy ArrayMergeStrategy) NavigationStrategy {
+	return &NavigationStrategyImpl{
+		visitedNodes: []*NodeContext{},
+		pathParser:   NewPathParser(),
+		followAlias: func(nodeContext NodeContext) bool {
+			return false
+		},
+		shouldOnlyDeeplyVisitLeaves: func(nodeContext NodeContext) bool {
+			return false
+		},
+		visit: func(nodeContext NodeContext) error {
+			return nil
+		},
+		shouldDeeplyTraverse: func(nodeContext NodeContext) bool {
+			if nodeContext.Node.Kind == yaml.SequenceNode && arrayMergeStrategy == OverwriteArrayMergeStrategy {
+				nodeContext.IsMiddleNode = false
+				return false
+			}
+
+			var isInArray = false
+			if len(nodeContext.PathStack) > 0 {
+				var lastElement = nodeContext.PathStack[len(nodeContext.PathStack)-1]
+				switch lastElement.(type) {
+				case int:
+					isInArray = true
+				default:
+					isInArray = false
+				}
+			}
+			return arrayMergeStrategy == UpdateArrayMergeStrategy || !isInArray
+		},
+	}
+}

pkg/yqlib/read_navigation_strategy.go

@@ -1,24 +1,11 @@
 package yqlib
 
-func ReadNavigationStrategy(deeplyTraverseArrays bool) NavigationStrategy {
+func ReadNavigationStrategy() NavigationStrategy {
 	return &NavigationStrategyImpl{
 		visitedNodes: []*NodeContext{},
 		pathParser:   NewPathParser(),
 		visit: func(nodeContext NodeContext) error {
 			return nil
 		},
-		shouldDeeplyTraverse: func(nodeContext NodeContext) bool {
-			var isInArray = false
-			if len(nodeContext.PathStack) > 0 {
-				var lastElement = nodeContext.PathStack[len(nodeContext.PathStack)-1]
-				switch lastElement.(type) {
-				case int:
-					isInArray = true
-				default:
-					isInArray = false
-				}
-			}
-			return deeplyTraverseArrays || !isInArray
-		},
 	}
 }

pkg/yqlib/update_navigation_strategy.go

@@ -24,10 +24,12 @@ func UpdateNavigationStrategy(updateCommand UpdateCommand, autoCreate bool) Navi
 				node.Tag = changesToApply.Tag
 				node.Kind = changesToApply.Kind
 				node.Style = changesToApply.Style
-				node.Content = changesToApply.Content
+				if !updateCommand.DontUpdateNodeContent {
+					node.Content = changesToApply.Content
+				}
 				node.Anchor = changesToApply.Anchor
 				node.Alias = changesToApply.Alias
-				if !updateCommand.DontUpdateComments {
+				if updateCommand.CommentsMergeStrategy != IgnoreCommentsMergeStrategy {
 					node.HeadComment = changesToApply.HeadComment
 					node.LineComment = changesToApply.LineComment
 					node.FootComment = changesToApply.FootComment

release_instructions.txt

@@ -4,6 +4,7 @@
 - tag git with same version number
 - make sure local build passes
 - push tag to git
+  - 3.4.0, v3
   - git push --tags
 - make local xcompile (builds binaries for all platforms)
 

scripts/doctools.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-brew install mkdocs libyaml
-pip3 install markdown-include
-pip3 install mkdocs-material
-

scripts/publish.sh

@@ -32,5 +32,5 @@ upload() {
     done < <(find ./build -mindepth 1 -maxdepth 1 -print0)
 }
 
-release
+# release
 upload

scripts/xcompile.sh

@@ -3,10 +3,12 @@
 # This assumes that gonative and gox is installed as per the 'one time setup' instructions
 # at https://github.com/inconshreveable/gonative
 
-gox -ldflags "${LDFLAGS}" -output="build/yq_{{.OS}}_{{.Arch}}"
+
+CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -output="build/yq_{{.OS}}_{{.Arch}}"
 # include non-default linux builds too
-gox -ldflags "${LDFLAGS}" -os=linux  -output="build/yq_{{.OS}}_{{.Arch}}"
+CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -os=linux  -output="build/yq_{{.OS}}_{{.Arch}}"
 
 cd build
 rhash -r -a . -P -o checksums
 
+rhash --list-hashes > checksums_hashes_order

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '3.3.2'
+version: '3.4.1'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.