New line in docs for better readability

removed repeated word

.github/ISSUE_TEMPLATE/bug_report.md

@@ -10,6 +10,9 @@ assignees: ''
 **Describe the bug**
 A clear and concise description of what the bug is.
 
+version of yq: 
+operating system:
+
 **Input Yaml**
 Concise yaml document(s) (as simple as possible to show the bug)
 data1.yml:

.github/workflows/go.yml

@@ -7,10 +7,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.14
+    - name: Set up Go 1.15
       uses: actions/setup-go@v1
       with:
-        go-version: 1.14
+        go-version: 1.15
       id: go
 
     - name: Check out code into the Go module directory

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.14 as builder
+FROM golang:1.15 as builder
 
 WORKDIR /go/src/mikefarah/yq
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.14
+FROM golang:1.15
 
 COPY scripts/devtools.sh /opt/devtools.sh
 

README.md

@@ -18,13 +18,29 @@ V3 is officially out - if you've been using v2 and want/need to upgrade, checkou
 ```
 brew install yq
 ```
+
+### Windows:
+```
+choco install yq
+```
+Supported by @chillum (https://chocolatey.org/packages/yq)
+
+### Alpine Linux
+- Enable community repo by adding ```$MIRROR/alpine/v$VERSION/community``` to ```/etc/apk/repositories```
+- Update database index with ```apk update```
+- Install yq with ```apk add yq```
+
+Supported by Tuan Hoang
+https://pkgs.alpinelinux.org/package/edge/community/x86/yq
+
+
 ### Ubuntu and other Linux distros supporting `snap` packages:
 ```
 snap install yq
 ```
 
 #### Snap notes
-`yq` installs with with [_strict confinement_](https://docs.snapcraft.io/snap-confinement/6233) in snap, this means it doesn't have direct access to root files. To read root files you can:
+`yq` installs with [_strict confinement_](https://docs.snapcraft.io/snap-confinement/6233) in snap, this means it doesn't have direct access to root files. To read root files you can:
 
 ```
 sudo cat /etc/myfile | yq r - a.path
@@ -48,7 +64,7 @@ sudo add-apt-repository ppa:rmescandon/yq
 sudo apt update
 sudo apt install yq -y
 ```
-Supported by @rmescandon
+Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
 
 ### Go Get:
 ```
@@ -60,20 +76,20 @@ GO111MODULE=on go get github.com/mikefarah/yq/v3
 Oneshot use:
 
 ```bash
-docker run --rm -v ${PWD}:/workdir mikefarah/yq yq [flags] <command> FILE...
+docker run --rm -v "${PWD}":/workdir mikefarah/yq yq [flags] <command> FILE...
 ```
 
 Run commands interactively:
 
 ```bash
-docker run --rm -it -v ${PWD}:/workdir mikefarah/yq sh
+docker run --rm -it -v "${PWD}":/workdir mikefarah/yq sh
 ```
 
 It can be useful to have a bash function to avoid typing the whole docker command:
 
 ```bash
 yq() {
-  docker run --rm -i -v ${PWD}:/workdir mikefarah/yq yq $@
+  docker run --rm -i -v "${PWD}":/workdir mikefarah/yq yq "$@"
 }
 ```
 
@@ -94,6 +110,7 @@ yq() {
 - [Pipe data in by using '-'](https://mikefarah.gitbook.io/yq/commands/read#from-stdin)
 - [Merge](https://mikefarah.gitbook.io/yq/commands/merge) multiple yaml files with various options for [overriding](https://mikefarah.gitbook.io/yq/commands/merge#overwrite-values) and [appending](https://mikefarah.gitbook.io/yq/commands/merge#append-values-with-arrays)
 - Supports multiple documents in a single yaml file for [reading](https://mikefarah.gitbook.io/yq/commands/read#multiple-documents), [writing](https://mikefarah.gitbook.io/yq/commands/write-update#multiple-documents) and [merging](https://mikefarah.gitbook.io/yq/commands/merge#multiple-documents)
+- General shell completion scripts (bash/zsh/fish/powershell) (https://mikefarah.gitbook.io/yq/commands/shell-completion)
 
 ## [Usage](https://mikefarah.gitbook.io/yq/)
 
@@ -105,18 +122,19 @@ Usage:
   yq [command]
 
 Available Commands:
-  compare     yq x [--prettyPrint/-P] dataA.yaml dataB.yaml 'b.e(name==fr*).value'
+  compare          yq x [--prettyPrint/-P] dataA.yaml dataB.yaml 'b.e(name==fr*).value'
-  delete      yq d [--inplace/-i] [--doc/-d index] sample.yaml 'b.e(name==fred)'
+  delete           yq d [--inplace/-i] [--doc/-d index] sample.yaml 'b.e(name==fred)'
-  help        Help about any command
+  help             Help about any command
-  merge       yq m [--inplace/-i] [--doc/-d index] [--overwrite/-x] [--append/-a] sample.yaml sample2.yaml
+  merge            yq m [--inplace/-i] [--doc/-d index] [--overwrite/-x] [--append/-a] sample.yaml sample2.yaml
-  new         yq n [--script/-s script_file] a.b.c newValue
+  new              yq n [--script/-s script_file] a.b.c newValue
-  prefix      yq p [--inplace/-i] [--doc/-d index] sample.yaml a.b.c
+  prefix           yq p [--inplace/-i] [--doc/-d index] sample.yaml a.b.c
-  read        yq r [--printMode/-p pv] sample.yaml 'b.e(name==fr*).value'
+  read             yq r [--printMode/-p pv] sample.yaml 'b.e(name==fr*).value'
-  validate    yq v sample.yaml
+  shell-completion Generates shell completion scripts
-  write       yq w [--inplace/-i] [--script/-s script_file] [--doc/-d index] sample.yaml 'b.e(name==fr*).value' newValue
+  validate         yq v sample.yaml
+  write            yq w [--inplace/-i] [--script/-s script_file] [--doc/-d index] sample.yaml 'b.e(name==fr*).value' newValue
 
 Flags:
-  -C, --colors        print using colors
+  -C, --colors        print with colors
   -h, --help          help for yq
   -I, --indent int    sets indent level for output (default 2)
   -P, --prettyPrint   pretty print
@@ -126,3 +144,6 @@ Flags:
 
 Use "yq [command] --help" for more information about a command.
 ```
+
+## Known Issues
+- `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)

cmd/compare.go

@@ -31,6 +31,8 @@ yq x -d1 dataA.yaml dataB.yaml 'a.b.c'
 	cmdCompare.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
 	cmdCompare.PersistentFlags().StringVarP(&printMode, "printMode", "p", "v", "print mode (v (values, default), p (paths), pv (path and value pairs)")
 	cmdCompare.PersistentFlags().StringVarP(&defaultValue, "defaultValue", "D", "", "default value printed when there are no results")
+	cmdCompare.PersistentFlags().BoolVarP(&stripComments, "stripComments", "", false, "strip comments out before comparing")
+	cmdCompare.PersistentFlags().BoolVarP(&explodeAnchors, "explodeAnchors", "X", false, "explode anchors")
 	return cmdCompare
 }
 

cmd/compare_test.go

@@ -16,6 +16,55 @@ func TestCompareSameCmd(t *testing.T) {
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
+func TestCompareIgnoreCommentsCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "compare --stripComments ../examples/data1.yaml ../examples/data1-no-comments.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := ``
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestCompareDontIgnoreCommentsCmd(t *testing.T) {
+	forceOsExit = false
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "compare ../examples/data1.yaml ../examples/data1-no-comments.yaml")
+
+	expectedOutput := `-a: simple # just the best
++a: simple
+ b: [1, 2]
+ c:
+   test: 1
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestCompareExplodeAnchorsCommentsCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "compare --explodeAnchors ../examples/simple-anchor.yaml ../examples/simple-anchor-exploded.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := ``
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestCompareDontExplodeAnchorsCmd(t *testing.T) {
+	forceOsExit = false
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "compare ../examples/simple-anchor.yaml ../examples/simple-anchor-exploded.yaml")
+
+	expectedOutput := `-foo: &foo
++foo:
+   a: 1
+ foobar:
+-  !!merge <<: *foo
++  a: 1
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestCompareDifferentCmd(t *testing.T) {
 	forceOsExit = false
 	cmd := getRootCommand()
@@ -40,9 +89,9 @@ func TestComparePrettyCmd(t *testing.T) {
 	}
 	expectedOutput := ` a: simple # just the best
  b:
- - 1
+   - 1
--- 2
+-  - 2
-+- 3
++  - 3
  c:
    test: 1
 `

cmd/constant.go

@@ -10,12 +10,15 @@ var printMode = "v"
 var printLength = false
 var unwrapScalar = true
 var customStyle = ""
+var anchorName = ""
+var makeAlias = false
 var stripComments = false
 var collectIntoArray = false
 var writeInplace = false
 var writeScript = ""
 var sourceYamlFile = ""
 var outputToJSON = false
+var exitStatus = false
 var prettyPrint = false
 var explodeAnchors = false
 var colorsEnabled = false
@@ -23,7 +26,8 @@ var defaultValue = ""
 var indent = 2
 var overwriteFlag = false
 var autoCreateFlag = true
-var appendFlag = false
+var arrayMergeStrategyFlag = "update"
+var commentsMergeStrategyFlag = "setWhenBlank"
 var verbose = false
 var version = false
 var docIndex = "0"

cmd/delete_test.go

@@ -94,8 +94,31 @@ b:
 	expectedOutput := `a: 2
 b:
   hi:
-  - name: fred
+    - name: fred
-  - name: sam
+    - name: sam
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestDeleteDeepSplatArrayYaml(t *testing.T) {
+	content := `thing: 123
+b:
+ hi:
+  - thing: item1
+    name: fred
+`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("delete %s **.thing", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `b:
+  hi:
+    - name: fred
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }

cmd/merge.go

@@ -11,14 +11,14 @@ func createMergeCmd() *cobra.Command {
 	var cmdMerge = &cobra.Command{
 		Use:     "merge [initial_yaml_file] [additional_yaml_file]...",
 		Aliases: []string{"m"},
-		Short:   "yq m [--inplace/-i] [--doc/-d index] [--overwrite/-x] [--append/-a] sample.yaml sample2.yaml",
+		Short:   "yq m [--inplace/-i] [--doc/-d index] [--overwrite/-x] [--arrayMerge/-a strategy] sample.yaml sample2.yaml",
 		Example: `
 yq merge things.yaml other.yaml
 yq merge --inplace things.yaml other.yaml
 yq m -i things.yaml other.yaml
 yq m --overwrite things.yaml other.yaml
 yq m -i -x things.yaml other.yaml
-yq m -i -a things.yaml other.yaml
+yq m -i -a=append things.yaml other.yaml
 yq m -i --autocreate=false things.yaml other.yaml
       `,
 		Long: `Updates the yaml file by adding/updating the path(s) and value(s) from additional yaml file(s).
@@ -32,7 +32,17 @@ If append flag is set then existing arrays will be merged with the arrays from e
 	cmdMerge.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace")
 	cmdMerge.PersistentFlags().BoolVarP(&overwriteFlag, "overwrite", "x", false, "update the yaml file by overwriting existing values")
 	cmdMerge.PersistentFlags().BoolVarP(&autoCreateFlag, "autocreate", "c", true, "automatically create any missing entries")
-	cmdMerge.PersistentFlags().BoolVarP(&appendFlag, "append", "a", false, "update the yaml file by appending array values")
+	cmdMerge.PersistentFlags().StringVarP(&arrayMergeStrategyFlag, "arrays", "a", "update", `array merge strategy (update/append/overwrite)
+update: recursively update arrays by their index
+append: concatenate arrays together
+overwrite: replace arrays
+`)
+	cmdMerge.PersistentFlags().StringVarP(&commentsMergeStrategyFlag, "comments", "", "setWhenBlank", `comments merge strategy (setWhenBlank/ignore/append/overwrite)
+setWhenBlank: set comment if the original document has no comment at that node
+ignore: leave comments as-is in the original
+append: append comments together
+overwrite: overwrite comments completely
+`)
 	cmdMerge.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
 	return cmdMerge
 }
@@ -41,9 +51,9 @@ If append flag is set then existing arrays will be merged with the arrays from e
 * We don't deeply traverse arrays when appending a merge, instead we want to
 * append the entire array element.
  */
-func createReadFunctionForMerge() func(*yaml.Node) ([]*yqlib.NodeContext, error) {
+func createReadFunctionForMerge(arrayMergeStrategy yqlib.ArrayMergeStrategy) func(*yaml.Node) ([]*yqlib.NodeContext, error) {
 	return func(dataBucket *yaml.Node) ([]*yqlib.NodeContext, error) {
-		return lib.Get(dataBucket, "**", !appendFlag)
+		return lib.GetForMerge(dataBucket, "**", arrayMergeStrategy)
 	}
 }
 
@@ -53,19 +63,59 @@ func mergeProperties(cmd *cobra.Command, args []string) error {
 	if len(args) < 1 {
 		return errors.New("Must provide at least 1 yaml file")
 	}
+	var arrayMergeStrategy yqlib.ArrayMergeStrategy
+
+	switch arrayMergeStrategyFlag {
+	case "update":
+		arrayMergeStrategy = yqlib.UpdateArrayMergeStrategy
+	case "append":
+		arrayMergeStrategy = yqlib.AppendArrayMergeStrategy
+	case "overwrite":
+		arrayMergeStrategy = yqlib.OverwriteArrayMergeStrategy
+	default:
+		return errors.New("Array merge strategy must be one of: update/append/overwrite")
+	}
+
+	var commentsMergeStrategy yqlib.CommentsMergeStrategy
+
+	switch commentsMergeStrategyFlag {
+	case "setWhenBlank":
+		commentsMergeStrategy = yqlib.SetWhenBlankCommentsMergeStrategy
+	case "ignore":
+		commentsMergeStrategy = yqlib.IgnoreCommentsMergeStrategy
+	case "append":
+		commentsMergeStrategy = yqlib.AppendCommentsMergeStrategy
+	case "overwrite":
+		commentsMergeStrategy = yqlib.OverwriteCommentsMergeStrategy
+	default:
+		return errors.New("Comments merge strategy must be one of: setWhenBlank/ignore/append/overwrite")
+	}
 
 	if len(args) > 1 {
 		// first generate update commands from the file
 		var filesToMerge = args[1:]
 
 		for _, fileToMerge := range filesToMerge {
-			matchingNodes, errorProcessingFile := doReadYamlFile(fileToMerge, createReadFunctionForMerge(), false, 0)
+			matchingNodes, errorProcessingFile := doReadYamlFile(fileToMerge, createReadFunctionForMerge(arrayMergeStrategy), false, 0)
 			if errorProcessingFile != nil {
 				return errorProcessingFile
 			}
+			log.Debugf("finished reading for merge!")
 			for _, matchingNode := range matchingNodes {
-				mergePath := lib.MergePathStackToString(matchingNode.PathStack, appendFlag)
+				log.Debugf("matched node %v", lib.PathStackToString(matchingNode.PathStack))
-				updateCommands = append(updateCommands, yqlib.UpdateCommand{Command: "update", Path: mergePath, Value: matchingNode.Node, Overwrite: overwriteFlag})
+				yqlib.DebugNode(matchingNode.Node)
+			}
+			for _, matchingNode := range matchingNodes {
+				mergePath := lib.MergePathStackToString(matchingNode.PathStack, arrayMergeStrategy)
+				updateCommands = append(updateCommands, yqlib.UpdateCommand{
+					Command:               "merge",
+					Path:                  mergePath,
+					Value:                 matchingNode.Node,
+					Overwrite:             overwriteFlag,
+					CommentsMergeStrategy: commentsMergeStrategy,
+					// dont update the content for nodes midway, only leaf nodes
+					DontUpdateNodeContent: matchingNode.IsMiddleNode && (arrayMergeStrategy != yqlib.OverwriteArrayMergeStrategy || matchingNode.Node.Kind != yaml.SequenceNode),
+				})
 			}
 		}
 	}

cmd/merge_test.go

@@ -60,7 +60,7 @@ func TestMergeOverwriteCmd(t *testing.T) {
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `a: other # better than the original
+	expectedOutput := `a: other # just the best
 b: [3, 4]
 c:
   test: 1
@@ -68,9 +68,36 @@ c:
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
+func TestMergeOverwriteDeepExampleCmd(t *testing.T) {
+	content := `c:
+  test: 1
+  thing: whatever
+`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	mergeContent := `c:
+  test: 5
+`
+	mergeFilename := test.WriteTempYamlFile(mergeContent)
+	defer test.RemoveTempYamlFile(mergeFilename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("merge --autocreate=false --overwrite %s %s", filename, mergeFilename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `c:
+  test: 5
+  thing: whatever
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestMergeAppendCmd(t *testing.T) {
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "merge --autocreate=false --append ../examples/data1.yaml ../examples/data2.yaml")
+	result := test.RunCmd(cmd, "merge --autocreate=false --arrays=append ../examples/data1.yaml ../examples/data2.yaml")
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
@@ -96,26 +123,69 @@ func TestMergeAppendArraysCmd(t *testing.T) {
 	defer test.RemoveTempYamlFile(mergeFilename)
 
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, fmt.Sprintf("merge --append -d* %s %s", filename, mergeFilename))
+	result := test.RunCmd(cmd, fmt.Sprintf("merge --arrays=append -d* %s %s", filename, mergeFilename))
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
 	expectedOutput := `people:
-- name: Barry
+  - name: Barry
-  age: 21
+    age: 21
-- name: Roger
+  - name: Roger
-  age: 44
+    age: 44
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeAliasArraysCmd(t *testing.T) {
+	content := `
+vars:
+  variable1: &var1 cat
+
+usage:
+  value1: *var1
+  valueAnother: *var1
+  valuePlain: thing
+`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	mergeContent := `
+vars:
+  variable2: &var2 puppy
+
+usage:
+  value2: *var2
+  valueAnother: *var2
+  valuePlain: *var2
+`
+
+	mergeFilename := test.WriteTempYamlFile(mergeContent)
+	defer test.RemoveTempYamlFile(mergeFilename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("merge -x %s %s", filename, mergeFilename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `vars:
+  variable1: &var1 cat
+  variable2: &var2 puppy
+usage:
+  value1: *var1
+  valueAnother: *var2
+  valuePlain: *var2
+  value2: *var2
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
 func TestMergeOverwriteAndAppendCmd(t *testing.T) {
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "merge --autocreate=false --append --overwrite ../examples/data1.yaml ../examples/data2.yaml")
+	result := test.RunCmd(cmd, "merge --autocreate=false --arrays=append --overwrite ../examples/data1.yaml ../examples/data2.yaml")
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `a: other # better than the original
+	expectedOutput := `a: other # just the best
 b: [1, 2, 3, 4]
 c:
   test: 1
@@ -123,13 +193,148 @@ c:
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
-func TestMergeArraysCmd(t *testing.T) {
+var commentContentA = `
+a: valueA1 # commentA1
+b: valueB1
+`
+
+var commentContentB = `
+a: valueA2 # commentA2
+b: valueB2 # commentB2
+c: valueC2 # commentC2
+`
+
+func TestMergeCommentsSetWhenBlankCmd(t *testing.T) {
+	filename := test.WriteTempYamlFile(commentContentA)
+	defer test.RemoveTempYamlFile(filename)
+
+	mergeFilename := test.WriteTempYamlFile(commentContentB)
+	defer test.RemoveTempYamlFile(mergeFilename)
+
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "merge --append ../examples/sample_array.yaml ../examples/sample_array_2.yaml")
+	result := test.RunCmd(cmd, fmt.Sprintf("merge --comments=setWhenBlank %s %s", filename, mergeFilename))
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `[1, 2, 3, 4, 5]
+
+	expectedOutput := `a: valueA1 # commentA1
+b: valueB1 # commentB2
+c: valueC2 # commentC2
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeCommentsIgnoreCmd(t *testing.T) {
+	filename := test.WriteTempYamlFile(commentContentA)
+	defer test.RemoveTempYamlFile(filename)
+
+	mergeFilename := test.WriteTempYamlFile(commentContentB)
+	defer test.RemoveTempYamlFile(mergeFilename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("merge --comments=ignore %s %s", filename, mergeFilename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `a: valueA1 # commentA1
+b: valueB1
+c: valueC2
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeCommentsAppendCmd(t *testing.T) {
+	filename := test.WriteTempYamlFile(commentContentA)
+	defer test.RemoveTempYamlFile(filename)
+
+	mergeFilename := test.WriteTempYamlFile(commentContentB)
+	defer test.RemoveTempYamlFile(mergeFilename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("merge --comments=append %s %s", filename, mergeFilename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `a: valueA1 # commentA1 # commentA2
+b: valueB1 # commentB2
+c: valueC2 # commentC2
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeCommentsOverwriteCmd(t *testing.T) {
+	filename := test.WriteTempYamlFile(commentContentA)
+	defer test.RemoveTempYamlFile(filename)
+
+	mergeFilename := test.WriteTempYamlFile(commentContentB)
+	defer test.RemoveTempYamlFile(mergeFilename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("merge --comments=overwrite %s %s", filename, mergeFilename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `a: valueA1 # commentA2
+b: valueB1 # commentB2
+c: valueC2 # commentC2
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeOverwriteArraysTooCmd(t *testing.T) {
+	content := `a: simple # just the best
+b: [1, 2]
+c:
+  test: 1
+`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	mergeContent := `a: things
+b: [6]`
+	mergeFilename := test.WriteTempYamlFile(mergeContent)
+	defer test.RemoveTempYamlFile(mergeFilename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("merge --autocreate=false --arrays=overwrite --overwrite %s %s", filename, mergeFilename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `a: things # just the best
+b: [6]
+c:
+  test: 1
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeRootArraysCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "merge --arrays=append ../examples/sample_array.yaml ../examples/sample_array_2.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `- 1
+- 2
+- 3
+- 4
+- 5
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeOverwriteArraysCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "merge --arrays=overwrite ../examples/sample_array.yaml ../examples/sample_array_2.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `- 4
+- 5
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -145,9 +350,7 @@ func TestMergeCmd_Multi(t *testing.T) {
 another:
   document: here
 a: simple # just the best
-b:
+b: [1, 2]
-- 1
-- 2
 c:
   test: 1
 ---
@@ -316,9 +519,7 @@ func TestMergeAllowEmptyTargetCmd(t *testing.T) {
 		t.Error(result.Error)
 	}
 	expectedOutput := `a: simple # just the best
-b:
+b: [1, 2]
-- 1
-- 2
 c:
   test: 1
 `

cmd/new.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"github.com/mikefarah/yq/v3/pkg/yqlib"
-	errors "github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
 
@@ -29,16 +28,14 @@ Note that you can give a create script to perform more sophisticated yaml. This
 	cmdNew.PersistentFlags().StringVarP(&writeScript, "script", "s", "", "yaml script for creating yaml")
 	cmdNew.PersistentFlags().StringVarP(&customTag, "tag", "t", "", "set yaml tag (e.g. !!int)")
 	cmdNew.PersistentFlags().StringVarP(&customStyle, "style", "", "", "formatting style of the value: single, double, folded, flow, literal, tagged")
+	cmdNew.PersistentFlags().StringVarP(&anchorName, "anchorName", "", "", "anchor name")
+	cmdNew.PersistentFlags().BoolVarP(&makeAlias, "makeAlias", "", false, "create an alias using the value as the anchor name")
 	return cmdNew
 }
 
 func newProperty(cmd *cobra.Command, args []string) error {
 	var badArgsMessage = "Must provide <path_to_update> <value>"
-	if len(args) != 2 {
+	var updateCommands, updateCommandsError = readUpdateCommands(args, 2, badArgsMessage, false)
-		return errors.New(badArgsMessage)
-	}
-
-	var updateCommands, updateCommandsError = readUpdateCommands(args, 2, badArgsMessage)
 	if updateCommandsError != nil {
 		return updateCommandsError
 	}

cmd/new_test.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/mikefarah/yq/v3/test"
@@ -18,6 +19,48 @@ func TestNewCmd(t *testing.T) {
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
+func TestNewCmdScript(t *testing.T) {
+	updateScript := `- command: update
+  path: b.c
+  value: 7`
+	scriptFilename := test.WriteTempYamlFile(updateScript)
+	defer test.RemoveTempYamlFile(scriptFilename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("new --script %s", scriptFilename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `b:
+  c: 7
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestNewAnchorCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "new b.c 3 --anchorName=fred")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `b:
+  c: &fred 3
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestNewAliasCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "new b.c foo --makeAlias")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `b:
+  c: *foo
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestNewArrayCmd(t *testing.T) {
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, "new b[0] 3")
@@ -25,7 +68,7 @@ func TestNewArrayCmd(t *testing.T) {
 		t.Error(result.Error)
 	}
 	expectedOutput := `b:
-- 3
+  - 3
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }

cmd/prefix_test.go

@@ -41,8 +41,8 @@ func TestPrefixCmdArray(t *testing.T) {
 		t.Error(result.Error)
 	}
 	expectedOutput := `- d:
-  - b:
+    - b:
-      c: 3
+        c: 3
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }

cmd/read.go

@@ -31,6 +31,7 @@ yq r -- things.yaml '--key-starting-with-dashes.blah'
 	cmdRead.PersistentFlags().BoolVarP(&unwrapScalar, "unwrapScalar", "", true, "unwrap scalar, print the value with no quotes, colors or comments")
 	cmdRead.PersistentFlags().BoolVarP(&stripComments, "stripComments", "", false, "print yaml without any comments")
 	cmdRead.PersistentFlags().BoolVarP(&explodeAnchors, "explodeAnchors", "X", false, "explode anchors")
+	cmdRead.PersistentFlags().BoolVarP(&exitStatus, "exitStatus", "e", false, "set exit status if no matches are found")
 	return cmdRead
 }
 
@@ -50,7 +51,13 @@ func readProperty(cmd *cobra.Command, args []string) error {
 
 	matchingNodes, errorReadingStream := readYamlFile(args[0], path, updateAll, docIndexInt)
 
+	if exitStatus && len(matchingNodes) == 0 {
+		cmd.SilenceUsage = true
+		return errors.New("No matches found")
+	}
+
 	if errorReadingStream != nil {
+		cmd.SilenceUsage = true
 		return errorReadingStream
 	}
 	out := cmd.OutOrStdout()

cmd/read_test.go

@@ -17,6 +17,30 @@ func TestReadCmd(t *testing.T) {
 	test.AssertResult(t, "2\n", result.Output)
 }
 
+func TestReadCmdWithExitStatus(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read ../examples/sample.yaml b.c -e")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	test.AssertResult(t, "2\n", result.Output)
+}
+
+func TestReadCmdWithExitStatusNotExist(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read ../examples/sample.yaml caterpillar -e")
+	test.AssertResult(t, "No matches found", result.Error.Error())
+}
+
+func TestReadCmdNotExist(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read ../examples/sample.yaml caterpillar")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	test.AssertResult(t, "", result.Output)
+}
+
 func TestReadUnwrapCmd(t *testing.T) {
 
 	content := `b: 'frog' # my favourite`
@@ -70,6 +94,28 @@ func TestReadUnwrapJsonByDefaultCmd(t *testing.T) {
 	test.AssertResult(t, "\"frog\"\n", result.Output)
 }
 
+func TestReadOutputJsonNonStringKeysCmd(t *testing.T) {
+
+	content := `
+true: true
+5: 
+  null: 
+    0.1: deeply
+    false: things`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("read %s -j", filename))
+
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `{"5":{"null":{"0.1":"deeply","false":"things"}},"true":true}
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestReadWithAdvancedFilterCmd(t *testing.T) {
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, "read ../examples/sample.yaml b.e(name==sam).value")
@@ -871,17 +917,35 @@ func TestReadPrettyPrintCmd(t *testing.T) {
 b:
   c: 2
   d:
-  - 3
+    - 3
-  - 4
+    - 4
   e:
-  - name: fred
+    - name: fred
-    value: 3
+      value: 3
-  - name: sam
+    - name: sam
-    value: 4
+      value: 4
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
+func TestReadNotFoundWithExitStatus(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read ../examples/sample.yaml adsf -e")
+	if result.Error == nil {
+		t.Error("Expected command to fail")
+	}
+	expectedOutput := `No matches found`
+	test.AssertResult(t, expectedOutput, result.Error.Error())
+}
+
+func TestReadNotFoundWithoutExitStatus(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read ../examples/sample.yaml adsf")
+	if result.Error != nil {
+		t.Error("Expected command to succeed!")
+	}
+}
+
 func TestReadPrettyPrintWithIndentCmd(t *testing.T) {
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, "read -P -I4 ../examples/sample.json")
@@ -892,13 +956,13 @@ func TestReadPrettyPrintWithIndentCmd(t *testing.T) {
 b:
     c: 2
     d:
-      - 3
+        - 3
-      - 4
+        - 4
     e:
-      - name: fred
+        - name: fred
-        value: 3
+          value: 3
-      - name: sam
+        - name: sam
-        value: 4
+          value: 4
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -914,8 +978,8 @@ func TestReadCmd_ArrayYaml_NoPath(t *testing.T) {
   hosts: lalaland
   name: "Apply smth"
   roles:
-  - lala
+    - lala
-  - land
+    - land
   serial: 1
 - become: false
   gather_facts: true
@@ -934,8 +998,8 @@ gather_facts: false
 hosts: lalaland
 name: "Apply smth"
 roles:
-- lala
+  - lala
-- land
+  - land
 serial: 1
 `
 	test.AssertResult(t, expectedOutput, result.Output)
@@ -952,8 +1016,8 @@ gather_facts: false
 hosts: lalaland
 name: "Apply smth"
 roles:
-- lala
+  - lala
-- land
+  - land
 serial: 1
 become: false
 gather_facts: true
@@ -973,8 +1037,8 @@ func TestReadCmd_ArrayYaml_SplatWithKeyAndValueCmd(t *testing.T) {
   hosts: lalaland
   name: "Apply smth"
   roles:
-  - lala
+    - lala
-  - land
+    - land
   serial: 1
 '[1]':
   become: false
@@ -1165,6 +1229,26 @@ func TestReadBadDataCmd(t *testing.T) {
 	test.AssertResult(t, expectedOutput, result.Error.Error())
 }
 
+func TestReadDeepFromRootCmd(t *testing.T) {
+	content := `state:
+  country: 
+    city: foo
+`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("read %s (**.city==foo)", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `country:
+  city: foo
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestReadSplatPrefixCmd(t *testing.T) {
 	content := `a: 2
 b:

cmd/root.go

@@ -54,8 +54,7 @@ func New() *cobra.Command {
 		createDeleteCmd(),
 		createNewCmd(),
 		createMergeCmd(),
+		createBashCompletionCmd(rootCmd),
 	)
-	rootCmd.SetOutput(os.Stdout)
-
 	return rootCmd
 }

cmd/shell_completion.go

@@ -0,0 +1,57 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+var shellVariant = "bash"
+
+func createBashCompletionCmd(rootCmd *cobra.Command) *cobra.Command {
+	var completionCmd = &cobra.Command{
+		Use:   "shell-completion",
+		Short: "Generates shell completion scripts",
+		Long: `To load completion for:
+bash:
+	Run	
+	. <(yq shell-completion)
+	
+	To configure your bash shell to load completions for each session add to
+	your bashrc
+	
+	# ~/.bashrc or ~/.profile
+	. <(yq shell-completion)
+
+zsh:
+	The generated completion script should be put somewhere in your $fpath named _yq
+
+powershell:
+	Users need PowerShell version 5.0 or above, which comes with Windows 10 and 
+	can be downloaded separately for Windows 7 or 8.1. They can then write the 
+	completions to a file and source this file from their PowerShell profile, 
+	which is referenced by the $Profile environment variable.
+
+fish:
+	Save the output to a fish file and add it to your completions directory.
+
+	`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			switch shellVariant {
+			case "bash", "":
+				return rootCmd.GenBashCompletion(os.Stdout)
+			case "zsh":
+				return rootCmd.GenZshCompletion(os.Stdout)
+			case "fish":
+				return rootCmd.GenFishCompletion(os.Stdout, true)
+			case "powershell":
+				return rootCmd.GenPowerShellCompletion(os.Stdout)
+			default:
+				return fmt.Errorf("Unknown variant %v", shellVariant)
+			}
+		},
+	}
+	completionCmd.PersistentFlags().StringVarP(&shellVariant, "variation", "V", "", "shell variation: bash (default), zsh, fish, powershell")
+	return completionCmd
+}

cmd/utils.go

@@ -17,7 +17,7 @@ type readDataFn func(dataBucket *yaml.Node) ([]*yqlib.NodeContext, error)
 
 func createReadFunction(path string) func(*yaml.Node) ([]*yqlib.NodeContext, error) {
 	return func(dataBucket *yaml.Node) ([]*yqlib.NodeContext, error) {
-		return lib.Get(dataBucket, path, true)
+		return lib.Get(dataBucket, path)
 	}
 }
 
@@ -149,25 +149,91 @@ func writeString(writer io.Writer, txt string) error {
 	return errorWriting
 }
 
+func setIfNotThere(node *yaml.Node, key string, value *yaml.Node) {
+	for index := 0; index < len(node.Content); index = index + 2 {
+		keyNode := node.Content[index]
+		if keyNode.Value == key {
+			return
+		}
+	}
+	// need to add it to the map
+	mapEntryKey := yaml.Node{Value: key, Kind: yaml.ScalarNode}
+	node.Content = append(node.Content, &mapEntryKey)
+	node.Content = append(node.Content, value)
+}
+
+func applyAlias(node *yaml.Node, alias *yaml.Node) {
+	for index := 0; index < len(alias.Content); index = index + 2 {
+		keyNode := alias.Content[index]
+		log.Debugf("applying alias key %v", keyNode.Value)
+		valueNode := alias.Content[index+1]
+		setIfNotThere(node, keyNode.Value, valueNode)
+	}
+}
+
+func explodeNode(node *yaml.Node) error {
+	node.Anchor = ""
+	switch node.Kind {
+	case yaml.SequenceNode, yaml.DocumentNode:
+		for index, contentNode := range node.Content {
+			log.Debugf("exploding index %v", index)
+			errorInContent := explodeNode(contentNode)
+			if errorInContent != nil {
+				return errorInContent
+			}
+		}
+		return nil
+	case yaml.AliasNode:
+		log.Debugf("its an alias!")
+		node.Kind = node.Alias.Kind
+		node.Style = node.Alias.Style
+		node.Tag = node.Alias.Tag
+		node.Content = node.Alias.Content
+		node.Value = node.Alias.Value
+		node.Alias = nil
+		return nil
+	case yaml.MappingNode:
+		for index := 0; index < len(node.Content); index = index + 2 {
+			keyNode := node.Content[index]
+			valueNode := node.Content[index+1]
+			log.Debugf("traversing %v", keyNode.Value)
+			if keyNode.Value != "<<" {
+				errorInContent := explodeNode(valueNode)
+				if errorInContent != nil {
+					return errorInContent
+				}
+			} else {
+				if valueNode.Kind == yaml.SequenceNode {
+					log.Debugf("an alias merge list!")
+					for index := len(valueNode.Content) - 1; index >= 0; index = index - 1 {
+						aliasNode := valueNode.Content[index]
+						applyAlias(node, aliasNode.Alias)
+					}
+				} else {
+					log.Debugf("an alias merge!")
+					applyAlias(node, valueNode.Alias)
+				}
+				node.Content = append(node.Content[:index], node.Content[index+2:]...)
+				//replay that index, since the array is shorter now.
+				index = index - 2
+			}
+		}
+
+		return nil
+	default:
+		return nil
+	}
+}
+
 func explode(matchingNodes []*yqlib.NodeContext) error {
 	log.Debug("exploding nodes")
 	for _, nodeContext := range matchingNodes {
-		var targetNode = yaml.Node{Kind: nodeContext.Node.Kind}
+		log.Debugf("exploding %v", nodeContext.Head)
-		explodedNodes, errorRetrieving := lib.Get(nodeContext.Node, "**", true)
+		errorExplodingNode := explodeNode(nodeContext.Node)
-		if errorRetrieving != nil {
+		if errorExplodingNode != nil {
-			return errorRetrieving
+			return errorExplodingNode
 		}
-		for _, matchingNode := range explodedNodes {
-			mergePath := lib.MergePathStackToString(matchingNode.PathStack, appendFlag)
-			updateCommand := yqlib.UpdateCommand{Command: "update", Path: mergePath, Value: matchingNode.Node, Overwrite: overwriteFlag}
-			errorUpdating := lib.Update(&targetNode, updateCommand, true)
-			if errorUpdating != nil {
-				return errorUpdating
-			}
-		}
-		nodeContext.Node = &targetNode
 	}
-	log.Debug("done exploding nodes")
 	return nil
 }
 
@@ -357,6 +423,16 @@ func readAndUpdate(stdOut io.Writer, inputFile string, updateData updateDataFn)
 		if err != nil {
 			return err
 		}
+		// mkdir temp dir as some docker images does not have temp dir
+		_, err = os.Stat(os.TempDir())
+		if os.IsNotExist(err) {
+			err = os.Mkdir(os.TempDir(), 0700)
+			if err != nil {
+				return err
+			}
+		} else if err != nil {
+			return err
+		}
 		tempFile, err := ioutil.TempFile("", "temp")
 		if err != nil {
 			return err
@@ -401,7 +477,7 @@ type updateCommandParsed struct {
 	Value   yaml.Node
 }
 
-func readUpdateCommands(args []string, expectedArgs int, badArgsMessage string) ([]yqlib.UpdateCommand, error) {
+func readUpdateCommands(args []string, expectedArgs int, badArgsMessage string, allowNoValue bool) ([]yqlib.UpdateCommand, error) {
 	var updateCommands []yqlib.UpdateCommand = make([]yqlib.UpdateCommand, 0)
 	if writeScript != "" {
 		var parsedCommands = make([]updateCommandParsed, 0)
@@ -435,13 +511,14 @@ func readUpdateCommands(args []string, expectedArgs int, badArgsMessage string)
 		log.Debug("args %v", args)
 		log.Debug("path %v", args[expectedArgs-2])
 		log.Debug("Value %v", args[expectedArgs-1])
-		updateCommands[0] = yqlib.UpdateCommand{Command: "update", Path: args[expectedArgs-2], Value: valueParser.Parse(args[expectedArgs-1], customTag, customStyle), Overwrite: true}
+		value := valueParser.Parse(args[expectedArgs-1], customTag, customStyle, anchorName, makeAlias)
-	} else if len(args) == expectedArgs-1 {
+		updateCommands[0] = yqlib.UpdateCommand{Command: "update", Path: args[expectedArgs-2], Value: value, Overwrite: true, CommentsMergeStrategy: yqlib.IgnoreCommentsMergeStrategy}
+	} else if len(args) == expectedArgs-1 && allowNoValue {
 		// don't update the value
 		updateCommands = make([]yqlib.UpdateCommand, 1)
 		log.Debug("args %v", args)
 		log.Debug("path %v", args[expectedArgs-2])
-		updateCommands[0] = yqlib.UpdateCommand{Command: "update", Path: args[expectedArgs-2], Value: valueParser.Parse("", customTag, customStyle), Overwrite: true, DontUpdateNodeValue: true}
+		updateCommands[0] = yqlib.UpdateCommand{Command: "update", Path: args[expectedArgs-2], Value: valueParser.Parse("", customTag, customStyle, anchorName, makeAlias), Overwrite: true, DontUpdateNodeValue: true}
 	} else {
 		return nil, errors.New(badArgsMessage)
 	}

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "3.3.0"
+	Version = "3.3.4"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

cmd/write.go

@@ -47,11 +47,13 @@ format is list of update commands (update or delete) like so:
 	cmdWrite.PersistentFlags().StringVarP(&customTag, "tag", "t", "", "set yaml tag (e.g. !!int)")
 	cmdWrite.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
 	cmdWrite.PersistentFlags().StringVarP(&customStyle, "style", "", "", "formatting style of the value: single, double, folded, flow, literal, tagged")
+	cmdWrite.PersistentFlags().StringVarP(&anchorName, "anchorName", "", "", "anchor name")
+	cmdWrite.PersistentFlags().BoolVarP(&makeAlias, "makeAlias", "", false, "create an alias using the value as the anchor name")
 	return cmdWrite
 }
 
 func writeProperty(cmd *cobra.Command, args []string) error {
-	var updateCommands, updateCommandsError = readUpdateCommands(args, 3, "Must provide <filename> <path_to_update> <value>")
+	var updateCommands, updateCommandsError = readUpdateCommands(args, 3, "Must provide <filename> <path_to_update> <value>", true)
 	if updateCommandsError != nil {
 		return updateCommandsError
 	}

cmd/write_test.go

@@ -27,6 +27,24 @@ func TestWriteCmd(t *testing.T) {
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
+func TestWriteKeepCommentsCmd(t *testing.T) {
+	content := `b:
+  c: 3 # comment
+`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.c 7", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `b:
+  c: 7 # comment
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestWriteWithTaggedStyleCmd(t *testing.T) {
 	content := `b:
   c: dog
@@ -261,9 +279,9 @@ func TestWriteAutoCreateCmd(t *testing.T) {
 		t.Error(result.Error)
 	}
 	expectedOutput := `applications:
-- name: app
+  - name: app
-  env:
+    env:
-    hello: world
+      hello: world
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -476,8 +494,8 @@ func TestWriteCmd_Append(t *testing.T) {
 		t.Error(result.Error)
 	}
 	expectedOutput := `b:
-- foo
+  - foo
-- 7
+  - 7
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -508,8 +526,8 @@ func TestWriteCmd_AppendInlinePretty(t *testing.T) {
 		t.Error(result.Error)
 	}
 	expectedOutput := `b:
-- foo
+  - foo
-- 7
+  - 7
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -527,7 +545,7 @@ func TestWriteCmd_AppendEmptyArray(t *testing.T) {
 	}
 	expectedOutput := `a: 2
 b:
-- v
+  - v
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -546,8 +564,8 @@ func TestWriteCmd_SplatArray(t *testing.T) {
 		t.Error(result.Error)
 	}
 	expectedOutput := `b:
-- c: new
+  - c: new
-- c: new
+  - c: new
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }

debian/changelog

@@ -1,3 +1,20 @@
+yq (3.3.2) focal; urgency=medium
+
+  * Bug fix: existStatus bug (#459)
+  * Automatically makes a os temp directory if it does not exist (#461)
+
+ -- Roberto Mier Escandon <rmescandon@gmail.com>  Fri, 07 Aug 2020 18:53:01 +0200
+
+yq (3.3-0) focal; urgency=medium
+
+  * You can control string styles (quotes) using the new --style flag
+  * String values now always have quotes when outputting to json
+  * Negative array indices now traverse the array backwards
+  * Added a --stripComments flag to print yaml without any comments
+  * Bumped go to version 1.14
+
+ -- Roberto Mier Escandon <rmescandon@gmail.com>  Thu, 30 Apr 2020 20:45:44 +0200
+
 yq (3.1-2) eoan; urgency=medium
 
   * Bug fix: yq 3 was removing empty inline-style objects and arrays (#355)

debian/files

@@ -1 +0,0 @@
-yq_3.1-2_source.buildinfo devel optional

examples/data1-no-comments.yaml

@@ -0,0 +1,4 @@
+a: simple
+b: [1, 2]
+c:
+  test: 1

examples/sample2.yaml

@@ -1,2 +0,0 @@
-b:
-  c: things

examples/simple-anchor-exploded.yaml

@@ -0,0 +1,5 @@
+foo:
+  a: 1
+
+foobar: 
+  a: 1

go.mod

@@ -2,16 +2,16 @@ module github.com/mikefarah/yq/v3
 
 require (
 	github.com/fatih/color v1.9.0
-	github.com/goccy/go-yaml v1.4.3
+	github.com/goccy/go-yaml v1.8.1
 	github.com/kylelemons/godebug v1.1.0
-	github.com/mattn/go-colorable v0.1.6 // indirect
+	github.com/mattn/go-colorable v0.1.7 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 // indirect
+	golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f // indirect
-	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
-	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
+	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
 )
 
-go 1.14
+go 1.15

go.sum

@@ -29,8 +29,8 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/goccy/go-yaml v1.4.3 h1:+1jK1ost1TBEfWjciIMU8rciBq0poxurgS7XvLgQInM=
+github.com/goccy/go-yaml v1.8.1 h1:JuZRFlqLM5cWF6A+waL8AKVuCcqvKOuhJtUQI+L3ez0=
-github.com/goccy/go-yaml v1.4.3/go.mod h1:PsEEJ29nIFZL07P/c8dv4P6rQkVFFXafQee85U+ERHA=
+github.com/goccy/go-yaml v1.8.1/go.mod h1:wS4gNoLalDSJxo/SpngzPQ2BN4uuZVLCmbM4S3vd4+Y=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -64,8 +64,8 @@ github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgx
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=
+github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw=
-github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM=
@@ -144,19 +144,20 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 h1:opSr2sbRXk5X5/givKrrKj9HXxFpW2sdCiP8MJSKLQY=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f h1:Fqb3ao1hUmOR3GkUOg/Y+BadLwykBIzs5q8Ez2SbHyc=
-golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
@@ -175,7 +176,6 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

pkg/yqlib/data_navigator.go

@@ -22,12 +22,8 @@ func NewDataNavigator(NavigationStrategy NavigationStrategy) DataNavigator {
 }
 
 func (n *navigator) Traverse(value *yaml.Node, path []interface{}) error {
-	realValue := value
 	emptyArray := make([]interface{}, 0)
-	if realValue.Kind == yaml.DocumentNode {
+	log.Debugf("Traversing path %v", pathStackToString(path))
-		log.Debugf("its a document! returning the first child")
-		return n.doTraverse(value.Content[0], "", path, emptyArray)
-	}
 	return n.doTraverse(value, "", path, emptyArray)
 }
 
@@ -38,7 +34,8 @@ func (n *navigator) doTraverse(value *yaml.Node, head interface{}, tail []interf
 	var nodeContext = NewNodeContext(value, head, tail, pathStack)
 
 	var errorDeepSplatting error
-	if head == "**" && value.Kind != yaml.ScalarNode && n.navigationStrategy.ShouldDeeplyTraverse(nodeContext) {
+	// no need to deeply traverse the DocumentNode, as it's already covered by its first child.
+	if head == "**" && value.Kind != yaml.DocumentNode && value.Kind != yaml.ScalarNode && n.navigationStrategy.ShouldDeeplyTraverse(nodeContext) {
 		if len(pathStack) == 0 || pathStack[len(pathStack)-1] != "<<" {
 			errorDeepSplatting = n.recurse(value, head, tail, pathStack)
 		}
@@ -50,7 +47,11 @@ func (n *navigator) doTraverse(value *yaml.Node, head interface{}, tail []interf
 		return errorDeepSplatting
 	}
 
-	if len(tail) > 0 && value.Kind != yaml.ScalarNode {
+	if value.Kind == yaml.DocumentNode {
+		log.Debugf("its a document, diving into %v", head)
+		DebugNode(value)
+		return n.recurse(value, head, tail, pathStack)
+	} else if len(tail) > 0 && value.Kind != yaml.ScalarNode {
 		log.Debugf("diving into %v", tail[0])
 		DebugNode(value)
 		return n.recurse(value, tail[0], tail[1:], pathStack)
@@ -68,6 +69,17 @@ func (n *navigator) getOrReplace(original *yaml.Node, expectedKind yaml.Kind) *y
 
 func (n *navigator) recurse(value *yaml.Node, head interface{}, tail []interface{}, pathStack []interface{}) error {
 	log.Debug("recursing, processing %v, pathStack %v", head, pathStackToString(pathStack))
+
+	nodeContext := NewNodeContext(value, head, tail, pathStack)
+
+	if head == "**" && !n.navigationStrategy.ShouldOnlyDeeplyVisitLeaves(nodeContext) {
+		nodeContext.IsMiddleNode = true
+		errorVisitingDeeply := n.navigationStrategy.Visit(nodeContext)
+		if errorVisitingDeeply != nil {
+			return errorVisitingDeeply
+		}
+	}
+
 	switch value.Kind {
 	case yaml.MappingNode:
 		log.Debug("its a map with %v entries", len(value.Content)/2)
@@ -85,20 +97,22 @@ func (n *navigator) recurse(value *yaml.Node, head interface{}, tail []interface
 			if head == "+" {
 				return n.appendArray(value, head, tail, pathStack)
 			} else if len(value.Content) == 0 && head == "**" {
-				return n.navigationStrategy.Visit(NewNodeContext(value, head, tail, pathStack))
+				return n.navigationStrategy.Visit(nodeContext)
 			}
 			return n.splatArray(value, head, tail, pathStack)
 		}
 	case yaml.AliasNode:
 		log.Debug("its an alias!")
 		DebugNode(value.Alias)
-		if n.navigationStrategy.FollowAlias(NewNodeContext(value, head, tail, pathStack)) {
+		if n.navigationStrategy.FollowAlias(nodeContext) {
 			log.Debug("following the alias")
 			return n.recurse(value.Alias, head, tail, pathStack)
 		}
 		return nil
+	case yaml.DocumentNode:
+		return n.doTraverse(value.Content[0], head, tail, pathStack)
 	default:
-		return n.navigationStrategy.Visit(NewNodeContext(value, head, tail, pathStack))
+		return n.navigationStrategy.Visit(nodeContext)
 	}
 }
 

pkg/yqlib/delete_navigation_strategy.go

@@ -12,15 +12,12 @@ func DeleteNavigationStrategy(pathElementToDelete interface{}) NavigationStrateg
 		followAlias: func(nodeContext NodeContext) bool {
 			return false
 		},
-		autoCreateMap: func(nodeContext NodeContext) bool {
+		shouldOnlyDeeplyVisitLeaves: func(nodeContext NodeContext) bool {
 			return false
 		},
-		shouldDeeplyTraverse: func(nodeContext NodeContext) bool {
-			return true
-		},
 		visit: func(nodeContext NodeContext) error {
 			node := nodeContext.Node
-			log.Debug("need to find and delete %v in here", pathElementToDelete)
+			log.Debug("need to find and delete %v in here (%v)", pathElementToDelete, pathStackToString(nodeContext.PathStack))
 			DebugNode(node)
 			if node.Kind == yaml.SequenceNode {
 				newContent := deleteFromArray(parser, node.Content, nodeContext.PathStack, pathElementToDelete)

pkg/yqlib/encoder.go

@@ -58,6 +58,21 @@ type jsonEncoder struct {
 	encoder *json.Encoder
 }
 
+func mapKeysToStrings(node *yaml.Node) {
+
+	if node.Kind == yaml.MappingNode {
+		for index, child := range node.Content {
+			if index%2 == 0 { // its a map key
+				child.Tag = "!!str"
+			}
+		}
+	}
+
+	for _, child := range node.Content {
+		mapKeysToStrings(child)
+	}
+}
+
 func NewJsonEncoder(destination io.Writer, prettyPrint bool, indent int) Encoder {
 	var encoder = json.NewEncoder(destination)
 	var indentString = ""
@@ -73,6 +88,8 @@ func NewJsonEncoder(destination io.Writer, prettyPrint bool, indent int) Encoder
 
 func (je *jsonEncoder) Encode(node *yaml.Node) error {
 	var dataBucket interface{}
+	// firstly, convert all map keys to strings
+	mapKeysToStrings(node)
 	errorDecoding := node.Decode(&dataBucket)
 	if errorDecoding != nil {
 		return errorDecoding

pkg/yqlib/filter_matching_node_navigation_strategy.go

@@ -4,12 +4,6 @@ func FilterMatchingNodesNavigationStrategy(value string) NavigationStrategy {
 	return &NavigationStrategyImpl{
 		visitedNodes: []*NodeContext{},
 		pathParser:   NewPathParser(),
-		followAlias: func(nodeContext NodeContext) bool {
-			return true
-		},
-		autoCreateMap: func(nodeContext NodeContext) bool {
-			return false
-		},
 		visit: func(nodeContext NodeContext) error {
 			return nil
 		},

pkg/yqlib/lib.go

@@ -13,11 +13,13 @@ import (
 var log = logging.MustGetLogger("yq")
 
 type UpdateCommand struct {
-	Command             string
+	Command               string
-	Path                string
+	Path                  string
-	Value               *yaml.Node
+	Value                 *yaml.Node
-	Overwrite           bool
+	Overwrite             bool
-	DontUpdateNodeValue bool
+	DontUpdateNodeValue   bool
+	DontUpdateNodeContent bool
+	CommentsMergeStrategy CommentsMergeStrategy
 }
 
 func KindString(kind yaml.Kind) string {
@@ -48,21 +50,24 @@ func DebugNode(value *yaml.Node) {
 			log.Error("Error debugging node, %v", errorEncoding.Error())
 		}
 		encoder.Close()
-		log.Debug("Tag: %v, Kind: %v", value.Tag, KindString(value.Kind))
+		log.Debug("Tag: %v, Kind: %v, Anchor: %v", value.Tag, KindString(value.Kind), value.Anchor)
-		log.Debug("%v", buf.String())
+		log.Debug("Head Comment: %v", value.HeadComment)
+		log.Debug("Line Comment: %v", value.LineComment)
+		log.Debug("FootComment Comment: %v", value.FootComment)
+		log.Debug("\n%v", buf.String())
 	}
 }
 
 func pathStackToString(pathStack []interface{}) string {
-	return mergePathStackToString(pathStack, false)
+	return mergePathStackToString(pathStack, UpdateArrayMergeStrategy)
 }
 
-func mergePathStackToString(pathStack []interface{}, appendArrays bool) string {
+func mergePathStackToString(pathStack []interface{}, arrayMergeStrategy ArrayMergeStrategy) string {
 	var sb strings.Builder
 	for index, path := range pathStack {
 		switch path.(type) {
 		case int, int64:
-			if appendArrays {
+			if arrayMergeStrategy == AppendArrayMergeStrategy {
 				sb.WriteString("[+]")
 			} else {
 				sb.WriteString(fmt.Sprintf("[%v]", path))
@@ -93,9 +98,7 @@ func mergePathStackToString(pathStack []interface{}, appendArrays bool) string {
 			sb.WriteString(".")
 		}
 	}
-	var pathString = sb.String()
+	return sb.String()
-	log.Debug("got a path string: %v", pathString)
-	return pathString
 }
 
 func guessKind(head interface{}, tail []interface{}, guess yaml.Kind) yaml.Kind {
@@ -133,12 +136,13 @@ func guessKind(head interface{}, tail []interface{}, guess yaml.Kind) yaml.Kind
 }
 
 type YqLib interface {
-	Get(rootNode *yaml.Node, path string, deeplyTraverseArrays bool) ([]*NodeContext, error)
+	Get(rootNode *yaml.Node, path string) ([]*NodeContext, error)
+	GetForMerge(rootNode *yaml.Node, path string, arrayMergeStrategy ArrayMergeStrategy) ([]*NodeContext, error)
 	Update(rootNode *yaml.Node, updateCommand UpdateCommand, autoCreate bool) error
 	New(path string) yaml.Node
 
 	PathStackToString(pathStack []interface{}) string
-	MergePathStackToString(pathStack []interface{}, appendArrays bool) string
+	MergePathStackToString(pathStack []interface{}, arrayMergeStrategy ArrayMergeStrategy) string
 }
 
 type lib struct {
@@ -151,21 +155,28 @@ func NewYqLib() YqLib {
 	}
 }
 
-func (l *lib) Get(rootNode *yaml.Node, path string, deeplyTraverseArrays bool) ([]*NodeContext, error) {
+func (l *lib) Get(rootNode *yaml.Node, path string) ([]*NodeContext, error) {
 	var paths = l.parser.ParsePath(path)
-	navigationStrategy := ReadNavigationStrategy(deeplyTraverseArrays)
+	navigationStrategy := ReadNavigationStrategy()
 	navigator := NewDataNavigator(navigationStrategy)
 	error := navigator.Traverse(rootNode, paths)
 	return navigationStrategy.GetVisitedNodes(), error
+}
 
+func (l *lib) GetForMerge(rootNode *yaml.Node, path string, arrayMergeStrategy ArrayMergeStrategy) ([]*NodeContext, error) {
+	var paths = l.parser.ParsePath(path)
+	navigationStrategy := ReadForMergeNavigationStrategy(arrayMergeStrategy)
+	navigator := NewDataNavigator(navigationStrategy)
+	error := navigator.Traverse(rootNode, paths)
+	return navigationStrategy.GetVisitedNodes(), error
 }
 
 func (l *lib) PathStackToString(pathStack []interface{}) string {
 	return pathStackToString(pathStack)
 }
 
-func (l *lib) MergePathStackToString(pathStack []interface{}, appendArrays bool) string {
+func (l *lib) MergePathStackToString(pathStack []interface{}, arrayMergeStrategy ArrayMergeStrategy) string {
-	return mergePathStackToString(pathStack, appendArrays)
+	return mergePathStackToString(pathStack, arrayMergeStrategy)
 }
 
 func (l *lib) New(path string) yaml.Node {
@@ -181,6 +192,10 @@ func (l *lib) Update(rootNode *yaml.Node, updateCommand UpdateCommand, autoCreat
 		var paths = l.parser.ParsePath(updateCommand.Path)
 		navigator := NewDataNavigator(UpdateNavigationStrategy(updateCommand, autoCreate))
 		return navigator.Traverse(rootNode, paths)
+	case "merge":
+		var paths = l.parser.ParsePath(updateCommand.Path)
+		navigator := NewDataNavigator(MergeNavigationStrategy(updateCommand, autoCreate))
+		return navigator.Traverse(rootNode, paths)
 	case "delete":
 		var paths = l.parser.ParsePath(updateCommand.Path)
 		lastBit, newTail := paths[len(paths)-1], paths[:len(paths)-1]

pkg/yqlib/lib_test.go

@@ -30,7 +30,7 @@ func TestLib(t *testing.T) {
 		array[0] = "a"
 		array[1] = 0
 		array[2] = "b"
-		got := subject.MergePathStackToString(array, true)
+		got := subject.MergePathStackToString(array, AppendArrayMergeStrategy)
 		test.AssertResult(t, `a.[+].b`, got)
 	})
 

pkg/yqlib/merge_navigation_strategy.go

@@ -0,0 +1,103 @@
+package yqlib
+
+import "gopkg.in/yaml.v3"
+
+type ArrayMergeStrategy uint32
+
+const (
+	UpdateArrayMergeStrategy ArrayMergeStrategy = 1 << iota
+	OverwriteArrayMergeStrategy
+	AppendArrayMergeStrategy
+)
+
+type CommentsMergeStrategy uint32
+
+const (
+	SetWhenBlankCommentsMergeStrategy CommentsMergeStrategy = 1 << iota
+	IgnoreCommentsMergeStrategy
+	OverwriteCommentsMergeStrategy
+	AppendCommentsMergeStrategy
+)
+
+func MergeNavigationStrategy(updateCommand UpdateCommand, autoCreate bool) NavigationStrategy {
+	return &NavigationStrategyImpl{
+		visitedNodes: []*NodeContext{},
+		pathParser:   NewPathParser(),
+		followAlias: func(nodeContext NodeContext) bool {
+			return false
+		},
+		autoCreateMap: func(nodeContext NodeContext) bool {
+			return autoCreate
+		},
+		visit: func(nodeContext NodeContext) error {
+			node := nodeContext.Node
+			changesToApply := updateCommand.Value
+
+			if node.Kind == yaml.DocumentNode && changesToApply.Kind != yaml.DocumentNode {
+				// when the path is empty, it matches both the top level pseudo document node
+				// and the actual top level node (e.g. map/sequence/whatever)
+				// so when we are updating with no path, make sure we update the right node.
+				node = node.Content[0]
+			}
+
+			log.Debug("going to update")
+			DebugNode(node)
+			log.Debug("with")
+			DebugNode(changesToApply)
+
+			if updateCommand.Overwrite || node.Value == "" {
+				node.Value = changesToApply.Value
+				node.Tag = changesToApply.Tag
+				node.Kind = changesToApply.Kind
+				node.Style = changesToApply.Style
+				node.Anchor = changesToApply.Anchor
+				node.Alias = changesToApply.Alias
+
+				if !updateCommand.DontUpdateNodeContent {
+					node.Content = changesToApply.Content
+				}
+			} else {
+				log.Debug("skipping update as node already has value %v and overwriteFlag is ", node.Value, updateCommand.Overwrite)
+			}
+
+			switch updateCommand.CommentsMergeStrategy {
+			case OverwriteCommentsMergeStrategy:
+				node.HeadComment = changesToApply.HeadComment
+				node.LineComment = changesToApply.LineComment
+				node.FootComment = changesToApply.FootComment
+			case SetWhenBlankCommentsMergeStrategy:
+				if node.HeadComment == "" {
+					node.HeadComment = changesToApply.HeadComment
+				}
+				if node.LineComment == "" {
+					node.LineComment = changesToApply.LineComment
+				}
+				if node.FootComment == "" {
+					node.FootComment = changesToApply.FootComment
+				}
+			case AppendCommentsMergeStrategy:
+				if node.HeadComment == "" {
+					node.HeadComment = changesToApply.HeadComment
+				} else {
+					node.HeadComment = node.HeadComment + "\n" + changesToApply.HeadComment
+				}
+				if node.LineComment == "" {
+					node.LineComment = changesToApply.LineComment
+				} else {
+					node.LineComment = node.LineComment + " " + changesToApply.LineComment
+				}
+				if node.FootComment == "" {
+					node.FootComment = changesToApply.FootComment
+				} else {
+					node.FootComment = node.FootComment + "\n" + changesToApply.FootComment
+				}
+			default:
+			}
+
+			log.Debug("result")
+			DebugNode(node)
+
+			return nil
+		},
+	}
+}

pkg/yqlib/navigation_strategy.go

@@ -11,6 +11,9 @@ type NodeContext struct {
 	Head      interface{}
 	Tail      []interface{}
 	PathStack []interface{}
+	// middle nodes are nodes that match along the original path, but not a
+	// target match of the path. This is only relevant when ShouldOnlyDeeplyVisitLeaves is false.
+	IsMiddleNode bool
 }
 
 func NewNodeContext(node *yaml.Node, head interface{}, tail []interface{}, pathStack []interface{}) NodeContext {
@@ -35,19 +38,22 @@ type NavigationStrategy interface {
 	// we use it to match against the pathExpression in head.
 	ShouldTraverse(nodeContext NodeContext, nodeKey string) bool
 	ShouldDeeplyTraverse(nodeContext NodeContext) bool
+	// when deeply traversing, should we visit all matching nodes, or just leaves?
+	ShouldOnlyDeeplyVisitLeaves(NodeContext) bool
 	GetVisitedNodes() []*NodeContext
 	DebugVisitedNodes()
 	GetPathParser() PathParser
 }
 
 type NavigationStrategyImpl struct {
-	followAlias          func(nodeContext NodeContext) bool
+	followAlias                 func(nodeContext NodeContext) bool
-	autoCreateMap        func(nodeContext NodeContext) bool
+	autoCreateMap               func(nodeContext NodeContext) bool
-	visit                func(nodeContext NodeContext) error
+	visit                       func(nodeContext NodeContext) error
-	shouldVisitExtraFn   func(nodeContext NodeContext) bool
+	shouldVisitExtraFn          func(nodeContext NodeContext) bool
-	shouldDeeplyTraverse func(nodeContext NodeContext) bool
+	shouldDeeplyTraverse        func(nodeContext NodeContext) bool
-	visitedNodes         []*NodeContext
+	shouldOnlyDeeplyVisitLeaves func(nodeContext NodeContext) bool
-	pathParser           PathParser
+	visitedNodes                []*NodeContext
+	pathParser                  PathParser
 }
 
 func (ns *NavigationStrategyImpl) GetPathParser() PathParser {
@@ -59,15 +65,32 @@ func (ns *NavigationStrategyImpl) GetVisitedNodes() []*NodeContext {
 }
 
 func (ns *NavigationStrategyImpl) FollowAlias(nodeContext NodeContext) bool {
-	return ns.followAlias(nodeContext)
+	if ns.followAlias != nil {
+		return ns.followAlias(nodeContext)
+	}
+	return true
 }
 
 func (ns *NavigationStrategyImpl) AutoCreateMap(nodeContext NodeContext) bool {
-	return ns.autoCreateMap(nodeContext)
+	if ns.autoCreateMap != nil {
+		return ns.autoCreateMap(nodeContext)
+	}
+	return false
 }
 
 func (ns *NavigationStrategyImpl) ShouldDeeplyTraverse(nodeContext NodeContext) bool {
-	return ns.shouldDeeplyTraverse(nodeContext)
+	if ns.shouldDeeplyTraverse != nil {
+		return ns.shouldDeeplyTraverse(nodeContext)
+	}
+	return true
+}
+
+func (ns *NavigationStrategyImpl) ShouldOnlyDeeplyVisitLeaves(nodeContext NodeContext) bool {
+	if ns.shouldOnlyDeeplyVisitLeaves != nil {
+		return ns.shouldOnlyDeeplyVisitLeaves(nodeContext)
+	}
+	return true
+
 }
 
 func (ns *NavigationStrategyImpl) ShouldTraverse(nodeContext NodeContext, nodeKey string) bool {

pkg/yqlib/read_for_merge_navigation_strategy.go

@@ -0,0 +1,37 @@
+package yqlib
+
+import "gopkg.in/yaml.v3"
+
+func ReadForMergeNavigationStrategy(arrayMergeStrategy ArrayMergeStrategy) NavigationStrategy {
+	return &NavigationStrategyImpl{
+		visitedNodes: []*NodeContext{},
+		pathParser:   NewPathParser(),
+		followAlias: func(nodeContext NodeContext) bool {
+			return false
+		},
+		shouldOnlyDeeplyVisitLeaves: func(nodeContext NodeContext) bool {
+			return false
+		},
+		visit: func(nodeContext NodeContext) error {
+			return nil
+		},
+		shouldDeeplyTraverse: func(nodeContext NodeContext) bool {
+			if nodeContext.Node.Kind == yaml.SequenceNode && arrayMergeStrategy == OverwriteArrayMergeStrategy {
+				nodeContext.IsMiddleNode = false
+				return false
+			}
+
+			var isInArray = false
+			if len(nodeContext.PathStack) > 0 {
+				var lastElement = nodeContext.PathStack[len(nodeContext.PathStack)-1]
+				switch lastElement.(type) {
+				case int:
+					isInArray = true
+				default:
+					isInArray = false
+				}
+			}
+			return arrayMergeStrategy == UpdateArrayMergeStrategy || !isInArray
+		},
+	}
+}

pkg/yqlib/read_navigation_strategy.go

@@ -1,30 +1,11 @@
 package yqlib
 
-func ReadNavigationStrategy(deeplyTraverseArrays bool) NavigationStrategy {
+func ReadNavigationStrategy() NavigationStrategy {
 	return &NavigationStrategyImpl{
 		visitedNodes: []*NodeContext{},
 		pathParser:   NewPathParser(),
-		followAlias: func(nodeContext NodeContext) bool {
-			return true
-		},
-		autoCreateMap: func(nodeContext NodeContext) bool {
-			return false
-		},
 		visit: func(nodeContext NodeContext) error {
 			return nil
 		},
-		shouldDeeplyTraverse: func(nodeContext NodeContext) bool {
-			var isInArray = false
-			if len(nodeContext.PathStack) > 0 {
-				var lastElement = nodeContext.PathStack[len(nodeContext.PathStack)-1]
-				switch lastElement.(type) {
-				case int:
-					isInArray = true
-				default:
-					isInArray = false
-				}
-			}
-			return deeplyTraverseArrays || !isInArray
-		},
 	}
 }

pkg/yqlib/update_navigation_strategy.go

@@ -10,9 +10,6 @@ func UpdateNavigationStrategy(updateCommand UpdateCommand, autoCreate bool) Navi
 		autoCreateMap: func(nodeContext NodeContext) bool {
 			return autoCreate
 		},
-		shouldDeeplyTraverse: func(nodeContext NodeContext) bool {
-			return true
-		},
 		visit: func(nodeContext NodeContext) error {
 			node := nodeContext.Node
 			changesToApply := updateCommand.Value
@@ -27,10 +24,16 @@ func UpdateNavigationStrategy(updateCommand UpdateCommand, autoCreate bool) Navi
 				node.Tag = changesToApply.Tag
 				node.Kind = changesToApply.Kind
 				node.Style = changesToApply.Style
-				node.Content = changesToApply.Content
+				if !updateCommand.DontUpdateNodeContent {
-				node.HeadComment = changesToApply.HeadComment
+					node.Content = changesToApply.Content
-				node.LineComment = changesToApply.LineComment
+				}
-				node.FootComment = changesToApply.FootComment
+				node.Anchor = changesToApply.Anchor
+				node.Alias = changesToApply.Alias
+				if updateCommand.CommentsMergeStrategy != IgnoreCommentsMergeStrategy {
+					node.HeadComment = changesToApply.HeadComment
+					node.LineComment = changesToApply.LineComment
+					node.FootComment = changesToApply.FootComment
+				}
 			} else {
 				log.Debug("skipping update as node already has value %v and overwriteFlag is ", node.Value, updateCommand.Overwrite)
 			}

pkg/yqlib/value_parser.go

@@ -5,7 +5,7 @@ import (
 )
 
 type ValueParser interface {
-	Parse(argument string, customTag string, customStyle string) *yaml.Node
+	Parse(argument string, customTag string, customStyle string, anchorName string, createAlias bool) *yaml.Node
 }
 
 type valueParser struct {
@@ -15,7 +15,7 @@ func NewValueParser() ValueParser {
 	return &valueParser{}
 }
 
-func (v *valueParser) Parse(argument string, customTag string, customStyle string) *yaml.Node {
+func (v *valueParser) Parse(argument string, customTag string, customStyle string, anchorName string, createAlias bool) *yaml.Node {
 	var style yaml.Style
 	if customStyle == "tagged" {
 		style = yaml.TaggedStyle
@@ -27,12 +27,20 @@ func (v *valueParser) Parse(argument string, customTag string, customStyle strin
 		style = yaml.LiteralStyle
 	} else if customStyle == "folded" {
 		style = yaml.FoldedStyle
+	} else if customStyle == "flow" {
+		style = yaml.FlowStyle
 	} else if customStyle != "" {
 		log.Error("Unknown style %v, ignoring", customStyle)
 	}
-
 	if argument == "[]" {
 		return &yaml.Node{Tag: "!!seq", Kind: yaml.SequenceNode, Style: style}
 	}
-	return &yaml.Node{Value: argument, Tag: customTag, Kind: yaml.ScalarNode, Style: style}
+
+	kind := yaml.ScalarNode
+
+	if createAlias {
+		kind = yaml.AliasNode
+	}
+
+	return &yaml.Node{Value: argument, Tag: customTag, Kind: kind, Style: style, Anchor: anchorName}
 }

pkg/yqlib/value_parser_test.go

@@ -16,12 +16,13 @@ var parseStyleTests = []struct {
 	{"double", yaml.DoubleQuotedStyle},
 	{"single", yaml.SingleQuotedStyle},
 	{"folded", yaml.FoldedStyle},
+	{"flow", yaml.FlowStyle},
 	{"literal", yaml.LiteralStyle},
 }
 
 func TestValueParserStyleTag(t *testing.T) {
 	for _, tt := range parseStyleTests {
-		actual := NewValueParser().Parse("cat", "", tt.customStyle)
+		actual := NewValueParser().Parse("cat", "", tt.customStyle, "", false)
 		test.AssertResultWithContext(t, tt.expectedStyle, actual.Style, tt.customStyle)
 	}
 }
@@ -39,7 +40,7 @@ var parseValueTests = []struct {
 
 func TestValueParserParse(t *testing.T) {
 	for _, tt := range parseValueTests {
-		actual := NewValueParser().Parse(tt.argument, tt.customTag, "")
+		actual := NewValueParser().Parse(tt.argument, tt.customTag, "", "", false)
 		test.AssertResultWithContext(t, tt.argument, actual.Value, tt.testDescription)
 		test.AssertResultWithContext(t, tt.expectedTag, actual.Tag, tt.testDescription)
 		test.AssertResult(t, yaml.ScalarNode, actual.Kind)
@@ -47,7 +48,18 @@ func TestValueParserParse(t *testing.T) {
 }
 
 func TestValueParserParseEmptyArray(t *testing.T) {
-	actual := NewValueParser().Parse("[]", "", "")
+	actual := NewValueParser().Parse("[]", "", "", "", false)
 	test.AssertResult(t, "!!seq", actual.Tag)
 	test.AssertResult(t, yaml.SequenceNode, actual.Kind)
 }
+
+func TestValueParserParseAlias(t *testing.T) {
+	actual := NewValueParser().Parse("bob", "", "", "", true)
+	test.AssertResult(t, "bob", actual.Value)
+	test.AssertResult(t, yaml.AliasNode, actual.Kind)
+}
+
+func TestValueParserAnchorname(t *testing.T) {
+	actual := NewValueParser().Parse("caterpillar", "", "", "foo", false)
+	test.AssertResult(t, "foo", actual.Anchor)
+}

scripts/doctools.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-brew install mkdocs libyaml
-pip3 install markdown-include
-pip3 install mkdocs-material
-

scripts/xcompile.sh

@@ -3,9 +3,10 @@
 # This assumes that gonative and gox is installed as per the 'one time setup' instructions
 # at https://github.com/inconshreveable/gonative
 
-gox -ldflags "${LDFLAGS}" -output="build/yq_{{.OS}}_{{.Arch}}"
+
+CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -output="build/yq_{{.OS}}_{{.Arch}}"
 # include non-default linux builds too
-gox -ldflags "${LDFLAGS}" -os=linux  -output="build/yq_{{.OS}}_{{.Arch}}"
+CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -os=linux  -output="build/yq_{{.OS}}_{{.Arch}}"
 
 cd build
 rhash -r -a . -P -o checksums

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '3.3.0'
+version: '3.3.4'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.
@@ -16,7 +16,7 @@ apps:
 parts:
   yq:
     plugin: go
-    go-channel: 1.14/stable
+    go-channel: 1.15/stable
     source: .
     source-type: git
     go-importpath: github.com/mikefarah/yq

yq.go

@@ -4,14 +4,11 @@ import (
 	"os"
 
 	command "github.com/mikefarah/yq/v3/cmd"
-	logging "gopkg.in/op/go-logging.v1"
 )
 
 func main() {
 	cmd := command.New()
-	log := logging.MustGetLogger("yq")
 	if err := cmd.Execute(); err != nil {
-		log.Error(err.Error())
 		os.Exit(1)
 	}
 }