Fixed explode for aliases to scalars

The program generates a path for every leaf node in the
file-to-be-merged. It does not escape them if they contain a dot, as
the path-expressions document mentions is necessary.

Add in a test for this condition. Verified it fails without the fix.

README.md

@@ -12,6 +12,10 @@ V3 is officially out - if you've been using v2 and want/need to upgrade, checkou
 
 ## Install
 
+### Download the latest binary
+
+[Here](https://github.com/mikefarah/yq/releases/latest)
+
 ### On MacOS:
 ```
 brew install yq
@@ -45,7 +49,7 @@ sudo add-apt-repository ppa:rmescandon/yq
 sudo apt update
 sudo apt install yq -y
 ```
-### or, [Download latest binary](https://github.com/mikefarah/yq/releases/latest) or alternatively:
+### Go Get:
 ```
 GO111MODULE=on go get github.com/mikefarah/yq/v3
 ```
@@ -78,7 +82,9 @@ yq() {
 - [List matching paths of a given path expression](https://mikefarah.gitbook.io/yq/commands/read#path-only)
 - Update a yaml file given a [path expression](https://mikefarah.gitbook.io/yq/commands/write-update#basic) or [script file](https://mikefarah.gitbook.io/yq/commands/write-update#basic)
 - Update creates any missing entries in the path on the fly
+- Deeply compare yaml files
 - Keeps yaml formatting and comments when updating
+- [Validate a yaml file](https://mikefarah.gitbook.io/yq/commands/validate)
 - Create a yaml file given a [deep path and value](https://mikefarah.gitbook.io/yq/commands/create#creating-a-simple-yaml-file) or a [script file](https://mikefarah.gitbook.io/yq/commands/create#creating-using-a-create-script)
 - [Prefix a path to a yaml file](https://mikefarah.gitbook.io/yq/commands/prefix)
 - [Convert to/from json to yaml](https://mikefarah.gitbook.io/yq/usage/convert)
@@ -96,19 +102,22 @@ Usage:
   yq [command]
 
 Available Commands:
+  compare     yq x [--prettyPrint/-P] dataA.yaml dataB.yaml 'b.e(name==fr*).value'
   delete      yq d [--inplace/-i] [--doc/-d index] sample.yaml 'b.e(name==fred)'
   help        Help about any command
   merge       yq m [--inplace/-i] [--doc/-d index] [--overwrite/-x] [--append/-a] sample.yaml sample2.yaml
   new         yq n [--script/-s script_file] a.b.c newValue
   prefix      yq p [--inplace/-i] [--doc/-d index] sample.yaml a.b.c
   read        yq r [--printMode/-p pv] sample.yaml 'b.e(name==fr*).value'
+  validate    yq v sample.yaml
   write       yq w [--inplace/-i] [--script/-s script_file] [--doc/-d index] sample.yaml 'b.e(name==fr*).value' newValue
 
 Flags:
-  -h, --help      help for yq
-  -j, --tojson    output as json
-  -v, --verbose   verbose mode
-  -V, --version   Print version information and quit
+  -h, --help          help for yq
+  -P, --prettyPrint   pretty print
+  -j, --tojson        output as json
+  -v, --verbose       verbose mode
+  -V, --version       Print version information and quit
 
 Use "yq [command] --help" for more information about a command.
 ```

cmd/commands_test.go

@@ -94,6 +94,54 @@ func TestReadCmd(t *testing.T) {
 	test.AssertResult(t, "2", result.Output)
 }
 
+func TestCompareCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "compare ../examples/data1.yaml ../examples/data3.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `-a: simple # just the best
+-b: [1, 2]
++a: "simple" # just the best
++b: [1, 3]
+ c:
+   test: 1
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestComparePrettyCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "compare -P ../examples/data1.yaml ../examples/data3.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := ` a: simple # just the best
+ b:
+ - 1
+-- 2
++- 3
+ c:
+   test: 1
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestComparePathsCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "compare -P -ppv ../examples/data1.yaml ../examples/data3.yaml **")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := ` a: simple # just the best
+ b.[0]: 1
+-b.[1]: 2
++b.[1]: 3
+ c.test: 1
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestValidateCmd(t *testing.T) {
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, "validate ../examples/sample.yaml b.c")
@@ -199,6 +247,32 @@ func TestReadAnchorsWithKeyAndValueCmd(t *testing.T) {
 	test.AssertResult(t, "foobar.a: 1\n", result.Output)
 }
 
+func TestReadAllAnchorsWithKeyAndValueCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -p pv ../examples/merge-anchor.yaml **")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `foo.a: original
+foo.thing: coolasdf
+foo.thirsty: yep
+bar.b: 2
+bar.thing: coconut
+bar.c: oldbar
+foobarList.c: newbar
+foobarList.b: 2
+foobarList.thing: coconut
+foobarList.a: original
+foobarList.thirsty: yep
+foobar.thirty: well beyond
+foobar.thing: ice
+foobar.c: 3
+foobar.a: original
+foobar.thirsty: yep
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestReadMergeAnchorsOriginalCmd(t *testing.T) {
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, "read ../examples/merge-anchor.yaml foobar.a")
@@ -208,6 +282,106 @@ func TestReadMergeAnchorsOriginalCmd(t *testing.T) {
 	test.AssertResult(t, "original", result.Output)
 }
 
+func TestReadMergeAnchorsExplodeJsonCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -j ../examples/merge-anchor.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `{"bar":{"b":2,"c":"oldbar","thing":"coconut"},"foo":{"a":"original","thing":"coolasdf","thirsty":"yep"},"foobar":{"a":"original","c":3,"thing":"ice","thirsty":"yep","thirty":"well beyond"},"foobarList":{"a":"original","b":2,"c":"newbar","thing":"coconut","thirsty":"yep"}}
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestReadMergeAnchorsExplodeSimpleCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -X ../examples/simple-anchor.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `foo:
+  a: 1
+foobar:
+  a: 1
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestReadMergeAnchorsExplodeSimpleValueCmd(t *testing.T) {
+	content := `value: &value-pointer the value
+pointer: *value-pointer`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("read -X %s pointer", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `the value`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestReadMergeAnchorsExplodeSimpleValueForValueCmd(t *testing.T) {
+	content := `value: &value-pointer the value
+pointer: *value-pointer`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("read -X %s value", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `the value`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestReadMergeAnchorsExplodeCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -X ../examples/merge-anchor.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `foo:
+  a: original
+  thing: coolasdf
+  thirsty: yep
+bar:
+  b: 2
+  thing: coconut
+  c: oldbar
+foobarList:
+  c: newbar
+  b: 2
+  thing: coconut
+  a: original
+  thirsty: yep
+foobar:
+  thirty: well beyond
+  thing: ice
+  c: 3
+  a: original
+  thirsty: yep
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestReadMergeAnchorsExplodeDeepCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -X ../examples/merge-anchor.yaml foobar")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `thirty: well beyond
+thing: ice
+c: 3
+a: original
+thirsty: yep
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestReadMergeAnchorsOverrideCmd(t *testing.T) {
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, "read ../examples/merge-anchor.yaml foobar.thing")
@@ -342,6 +516,76 @@ func TestReadCmd_ArrayYaml(t *testing.T) {
 	test.AssertResult(t, "false", result.Output)
 }
 
+func TestReadEmptyContentCmd(t *testing.T) {
+	content := ``
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("read %s", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := ``
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestReadEmptyContentWithDefaultValueCmd(t *testing.T) {
+	content := ``
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("read --defaultValue things %s", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `things`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestReadPrettyPrintCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -P ../examples/sample.json")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `a: Easy! as one two three
+b:
+  c: 2
+  d:
+  - 3
+  - 4
+  e:
+  - name: fred
+    value: 3
+  - name: sam
+    value: 4
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestReadPrettyPrintWithIndentCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -P -I4 ../examples/sample.json")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `a: Easy! as one two three
+b:
+    c: 2
+    d:
+      - 3
+      - 4
+    e:
+      - name: fred
+        value: 3
+      - name: sam
+        value: 4
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestReadCmd_ArrayYaml_NoPath(t *testing.T) {
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, "read ../examples/array.yaml")
@@ -530,23 +774,72 @@ func TestReadCmd_Verbose(t *testing.T) {
 	test.AssertResult(t, "2", result.Output)
 }
 
-// func TestReadCmd_ToJson(t *testing.T) {
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, "read -j ../examples/sample.yaml b.c")
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	test.AssertResult(t, "2\n", result.Output)
-// }
+func TestReadToJsonCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -j ../examples/sample.yaml b")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `{"c":2,"d":[3,4,5],"e":[{"name":"fred","value":3},{"name":"sam","value":4}]}
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
 
-// func TestReadCmd_ToJsonLong(t *testing.T) {
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, "read --tojson ../examples/sample.yaml b.c")
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	test.AssertResult(t, "2\n", result.Output)
-// }
+func TestReadToJsonPrettyCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -j -P ../examples/sample.yaml b")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `{
+  "c": 2,
+  "d": [
+    3,
+    4,
+    5
+  ],
+  "e": [
+    {
+      "name": "fred",
+      "value": 3
+    },
+    {
+      "name": "sam",
+      "value": 4
+    }
+  ]
+}
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestReadToJsonPrettyIndentCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -j -I4 -P ../examples/sample.yaml b")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `{
+    "c": 2,
+    "d": [
+        3,
+        4,
+        5
+    ],
+    "e": [
+        {
+            "name": "fred",
+            "value": 3
+        },
+        {
+            "name": "sam",
+            "value": 4
+        }
+    ]
+}
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
 
 func TestReadBadDataCmd(t *testing.T) {
 	content := `[!Whatever]`
@@ -908,6 +1201,22 @@ func TestWriteCmd(t *testing.T) {
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
+func TestWriteEmptyCmd(t *testing.T) {
+	content := ``
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.c 7", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `b:
+  c: 7
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestWriteCmdScript(t *testing.T) {
 	content := `b:
   c: 3
@@ -1106,6 +1415,38 @@ func TestWriteCmd_Append(t *testing.T) {
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
+func TestWriteCmd_AppendInline(t *testing.T) {
+	content := `b: [foo]`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("write %s b[+] 7", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `b: [foo, 7]
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestWriteCmd_AppendInlinePretty(t *testing.T) {
+	content := `b: [foo]`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("write %s -P b[+] 7", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `b:
+- foo
+- 7
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestWriteCmd_AppendEmptyArray(t *testing.T) {
 	content := `a: 2
 `
@@ -1206,6 +1547,22 @@ b:
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
+func TestDeleteDeepDoesNotExistCmd(t *testing.T) {
+	content := `a: 2`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("delete %s b.c", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `a: 2
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestDeleteSplatYaml(t *testing.T) {
 	content := `a: other
 b: [3, 4]
@@ -1213,7 +1570,7 @@ c:
   toast: leave
   test: 1
   tell: 1
-  taco: cool
+  tasty.taco: cool
 `
 	filename := test.WriteTempYamlFile(content)
 	defer test.RemoveTempYamlFile(filename)
@@ -1228,7 +1585,7 @@ c:
 b: [3, 4]
 c:
   toast: leave
-  taco: cool
+  tasty.taco: cool
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -1394,7 +1751,21 @@ c:
   test: 1
   toast: leave
   tell: 1
-  taco: cool
+  tasty.taco: cool
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeOneFileCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "merge ../examples/data1.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `a: simple # just the best
+b: [1, 2]
+c:
+  test: 1
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -1551,11 +1922,11 @@ apples: red
 
 func TestMergeCmd_Error(t *testing.T) {
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "merge ../examples/data1.yaml")
+	result := test.RunCmd(cmd, "merge")
 	if result.Error == nil {
 		t.Error("Expected command to fail due to missing arg")
 	}
-	expectedOutput := `Must provide at least 2 yaml files`
+	expectedOutput := `Must provide at least 1 yaml file`
 	test.AssertResult(t, expectedOutput, result.Error.Error())
 }
 
@@ -1595,29 +1966,35 @@ c:
   test: 1
   toast: leave
   tell: 1
-  taco: cool
+  tasty.taco: cool
 `
 	test.AssertResult(t, expectedOutput, gotOutput)
 	test.AssertResult(t, os.FileMode(int(0666)), info.Mode())
 }
 
-func TestMergeAllowEmptyCmd(t *testing.T) {
+func TestMergeAllowEmptyTargetCmd(t *testing.T) {
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "merge --allow-empty ../examples/data1.yaml ../examples/empty.yaml")
+	result := test.RunCmd(cmd, "merge ../examples/empty.yaml ../examples/data1.yaml")
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
 	expectedOutput := `a: simple # just the best
+b:
+- 1
+- 2
+c:
+  test: 1
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeAllowEmptyMergeCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "merge ../examples/data1.yaml ../examples/empty.yaml")
+	expectedOutput := `a: simple # just the best
 b: [1, 2]
 c:
   test: 1
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
-
-func TestMergeDontAllowEmptyCmd(t *testing.T) {
-	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "merge ../examples/data1.yaml ../examples/empty.yaml")
-	expectedOutput := `Could not process document index 0 as there are only 0 document(s)`
-	test.AssertResult(t, expectedOutput, result.Error.Error())
-}

cmd/compare.go

@@ -0,0 +1,76 @@
+package cmd
+
+import (
+	"bufio"
+	"bytes"
+	"strings"
+
+	"github.com/kylelemons/godebug/diff"
+	"github.com/mikefarah/yq/v3/pkg/yqlib"
+	errors "github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+func createCompareCmd() *cobra.Command {
+	var cmdCompare = &cobra.Command{
+		Use:     "compare [yaml_file_a] [yaml_file_b]",
+		Aliases: []string{"x"},
+		Short:   "yq x [--prettyPrint/-P] dataA.yaml dataB.yaml 'b.e(name==fr*).value'",
+		Example: `
+yq x - data2.yml # reads from stdin
+yq x -pp dataA.yaml dataB.yaml '**' # compare paths
+yq x -d1 dataA.yaml dataB.yaml 'a.b.c'
+`,
+		Long: "Deeply compares two yaml files, prints the difference. Use with prettyPrint flag to ignore formatting differences.",
+		RunE: compareDocuments,
+	}
+	cmdCompare.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
+	cmdCompare.PersistentFlags().StringVarP(&printMode, "printMode", "p", "v", "print mode (v (values, default), p (paths), pv (path and value pairs)")
+	cmdCompare.PersistentFlags().StringVarP(&defaultValue, "defaultValue", "D", "", "default value printed when there are no results")
+	return cmdCompare
+}
+
+func compareDocuments(cmd *cobra.Command, args []string) error {
+	var path = ""
+
+	if len(args) < 2 {
+		return errors.New("Must provide at 2 yaml files")
+	} else if len(args) > 2 {
+		path = args[2]
+	}
+
+	var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex()
+	if errorParsingDocIndex != nil {
+		return errorParsingDocIndex
+	}
+
+	var matchingNodesA []*yqlib.NodeContext
+	var matchingNodesB []*yqlib.NodeContext
+	var errorDoingThings error
+
+	matchingNodesA, errorDoingThings = readYamlFile(args[0], path, updateAll, docIndexInt)
+
+	if errorDoingThings != nil {
+		return errorDoingThings
+	}
+
+	matchingNodesB, errorDoingThings = readYamlFile(args[1], path, updateAll, docIndexInt)
+	if errorDoingThings != nil {
+		return errorDoingThings
+	}
+
+	var dataBufferA bytes.Buffer
+	var dataBufferB bytes.Buffer
+	errorDoingThings = printResults(matchingNodesA, bufio.NewWriter(&dataBufferA))
+	if errorDoingThings != nil {
+		return errorDoingThings
+	}
+	errorDoingThings = printResults(matchingNodesB, bufio.NewWriter(&dataBufferB))
+	if errorDoingThings != nil {
+		return errorDoingThings
+	}
+
+	cmd.Print(diff.Diff(strings.TrimSuffix(dataBufferA.String(), "\n"), strings.TrimSuffix(dataBufferB.String(), "\n")))
+	cmd.Print("\n")
+	return nil
+}

cmd/constant.go

@@ -10,9 +10,12 @@ var printMode = "v"
 var writeInplace = false
 var writeScript = ""
 var outputToJSON = false
+var prettyPrint = false
+var explodeAnchors = false
+var defaultValue = ""
+var indent = 2
 var overwriteFlag = false
 var autoCreateFlag = true
-var allowEmptyFlag = false
 var appendFlag = false
 var verbose = false
 var version = false

cmd/merge.go

@@ -1,8 +1,6 @@
 package cmd
 
 import (
-	"strings"
-
 	"github.com/mikefarah/yq/v3/pkg/yqlib"
 	errors "github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -34,27 +32,30 @@ If append flag is set then existing arrays will be merged with the arrays from e
 	cmdMerge.PersistentFlags().BoolVarP(&overwriteFlag, "overwrite", "x", false, "update the yaml file by overwriting existing values")
 	cmdMerge.PersistentFlags().BoolVarP(&autoCreateFlag, "autocreate", "c", true, "automatically create any missing entries")
 	cmdMerge.PersistentFlags().BoolVarP(&appendFlag, "append", "a", false, "update the yaml file by appending array values")
-	cmdMerge.PersistentFlags().BoolVarP(&allowEmptyFlag, "allow-empty", "e", false, "allow empty yaml files")
 	cmdMerge.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
 	return cmdMerge
 }
 
 func mergeProperties(cmd *cobra.Command, args []string) error {
-	if len(args) < 2 {
-		return errors.New("Must provide at least 2 yaml files")
-	}
-	// first generate update commands from the file
-	var filesToMerge = args[1:]
 	var updateCommands []yqlib.UpdateCommand = make([]yqlib.UpdateCommand, 0)
 
-	for _, fileToMerge := range filesToMerge {
-		matchingNodes, errorProcessingFile := readYamlFile(fileToMerge, "**", false, 0)
-		if errorProcessingFile != nil && (!allowEmptyFlag || !strings.HasPrefix(errorProcessingFile.Error(), "Could not process document index")) {
-			return errorProcessingFile
-		}
-		for _, matchingNode := range matchingNodes {
-			mergePath := lib.MergePathStackToString(matchingNode.PathStack, appendFlag)
-			updateCommands = append(updateCommands, yqlib.UpdateCommand{Command: "update", Path: mergePath, Value: matchingNode.Node, Overwrite: overwriteFlag})
+	if len(args) < 1 {
+		return errors.New("Must provide at least 1 yaml file")
+	}
+
+	if len(args) > 1 {
+		// first generate update commands from the file
+		var filesToMerge = args[1:]
+
+		for _, fileToMerge := range filesToMerge {
+			matchingNodes, errorProcessingFile := readYamlFile(fileToMerge, "**", false, 0)
+			if errorProcessingFile != nil {
+				return errorProcessingFile
+			}
+			for _, matchingNode := range matchingNodes {
+				mergePath := lib.MergePathStackToString(matchingNode.PathStack, appendFlag)
+				updateCommands = append(updateCommands, yqlib.UpdateCommand{Command: "update", Path: mergePath, Value: matchingNode.Node, Overwrite: overwriteFlag})
+			}
 		}
 	}
 

cmd/read.go

@@ -25,6 +25,8 @@ yq r -- things.yaml '--key-starting-with-dashes.blah'
 	}
 	cmdRead.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
 	cmdRead.PersistentFlags().StringVarP(&printMode, "printMode", "p", "v", "print mode (v (values, default), p (paths), pv (path and value pairs)")
+	cmdRead.PersistentFlags().StringVarP(&defaultValue, "defaultValue", "D", "", "default value printed when there are no results")
+	cmdRead.PersistentFlags().BoolVarP(&explodeAnchors, "explodeAnchors", "X", false, "explode anchors")
 	return cmdRead
 }
 
@@ -48,5 +50,5 @@ func readProperty(cmd *cobra.Command, args []string) error {
 		return errorReadingStream
 	}
 
-	return printResults(matchingNodes, cmd)
+	return printResults(matchingNodes, cmd.OutOrStdout())
 }

cmd/root.go

@@ -39,11 +39,14 @@ func New() *cobra.Command {
 	}
 
 	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose mode")
-	rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json")
+	rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json. By default it prints a json document in one line, use the prettyPrint flag to print a formatted doc.")
+	rootCmd.PersistentFlags().BoolVarP(&prettyPrint, "prettyPrint", "P", false, "pretty print")
+	rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output")
 	rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
 
 	rootCmd.AddCommand(
 		createReadCmd(),
+		createCompareCmd(),
 		createValidateCmd(),
 		createWriteCmd(),
 		createPrefixCmd(),

cmd/utils.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/mikefarah/yq/v3/pkg/yqlib"
 	errors "github.com/pkg/errors"
-	"github.com/spf13/cobra"
 	yaml "gopkg.in/yaml.v3"
 )
 
@@ -46,7 +45,7 @@ func readYamlFile(filename string, path string, updateAll bool, docIndexInt int)
 
 func handleEOF(updateAll bool, docIndexInt int, currentIndex int) error {
 	log.Debugf("done %v / %v", currentIndex, docIndexInt)
-	if !updateAll && currentIndex <= docIndexInt {
+	if !updateAll && currentIndex <= docIndexInt && docIndexInt != 0 {
 		return fmt.Errorf("Could not process document index %v as there are only %v document(s)", docIndex, currentIndex)
 	}
 	return nil
@@ -66,39 +65,100 @@ func appendDocument(originalMatchingNodes []*yqlib.NodeContext, dataBucket yaml.
 	return append(originalMatchingNodes, matchingNodes...), nil
 }
 
-func printValue(node *yaml.Node, cmd *cobra.Command) error {
+func printValue(node *yaml.Node, writer io.Writer) error {
 	if node.Kind == yaml.ScalarNode {
-		cmd.Print(node.Value)
-		return nil
+		_, errorWriting := writer.Write([]byte(node.Value))
+		return errorWriting
 	}
+	return printNode(node, writer)
+}
 
-	bufferedWriter := bufio.NewWriter(cmd.OutOrStdout())
-	defer safelyFlush(bufferedWriter)
-
+func printNode(node *yaml.Node, writer io.Writer) error {
 	var encoder yqlib.Encoder
 	if outputToJSON {
-		encoder = yqlib.NewJsonEncoder(bufferedWriter)
+		encoder = yqlib.NewJsonEncoder(writer, prettyPrint, indent)
 	} else {
-		encoder = yqlib.NewYamlEncoder(bufferedWriter)
+		encoder = yqlib.NewYamlEncoder(writer, indent)
 	}
-	if err := encoder.Encode(node); err != nil {
-		return err
+	return encoder.Encode(node)
+}
+
+func setStyle(matchingNodes []*yqlib.NodeContext, style yaml.Style) {
+	for _, nodeContext := range matchingNodes {
+		updateStyleOfNode(nodeContext.Node, style)
+	}
+}
+
+func updateStyleOfNode(node *yaml.Node, style yaml.Style) {
+	node.Style = style
+
+	for _, child := range node.Content {
+		updateStyleOfNode(child, style)
+	}
+}
+
+func writeString(writer io.Writer, txt string) error {
+	_, errorWriting := writer.Write([]byte(txt))
+	return errorWriting
+}
+
+func explode(matchingNodes []*yqlib.NodeContext) error {
+
+	for _, nodeContext := range matchingNodes {
+		var targetNode = yaml.Node{Kind: yaml.MappingNode}
+		explodedNodes, errorRetrieving := lib.Get(nodeContext.Node, "**")
+		if errorRetrieving != nil {
+			return errorRetrieving
+		}
+		for _, matchingNode := range explodedNodes {
+			mergePath := lib.MergePathStackToString(matchingNode.PathStack, appendFlag)
+			updateCommand := yqlib.UpdateCommand{Command: "update", Path: mergePath, Value: matchingNode.Node, Overwrite: overwriteFlag}
+			errorUpdating := lib.Update(&targetNode, updateCommand, true)
+			if errorUpdating != nil {
+				return errorUpdating
+			}
+		}
+		nodeContext.Node = &targetNode
 	}
 	return nil
 }
 
-func printResults(matchingNodes []*yqlib.NodeContext, cmd *cobra.Command) error {
-	if len(matchingNodes) == 0 {
-		log.Debug("no matching results, nothing to print")
-		return nil
+func printResults(matchingNodes []*yqlib.NodeContext, writer io.Writer) error {
+	if prettyPrint {
+		setStyle(matchingNodes, 0)
 	}
 
+	//always explode anchors when printing json
+	if explodeAnchors || outputToJSON {
+		errorExploding := explode(matchingNodes)
+		if errorExploding != nil {
+			return errorExploding
+		}
+	}
+
+	bufferedWriter := bufio.NewWriter(writer)
+	defer safelyFlush(bufferedWriter)
+
+	if len(matchingNodes) == 0 {
+		log.Debug("no matching results, nothing to print")
+		if defaultValue != "" {
+			return writeString(bufferedWriter, defaultValue)
+		}
+		return nil
+	}
+	var errorWriting error
 	for index, mappedDoc := range matchingNodes {
 		switch printMode {
 		case "p":
-			cmd.Print(lib.PathStackToString(mappedDoc.PathStack))
+			errorWriting = writeString(bufferedWriter, lib.PathStackToString(mappedDoc.PathStack))
+			if errorWriting != nil {
+				return errorWriting
+			}
 			if index < len(matchingNodes)-1 {
-				cmd.Print("\n")
+				errorWriting = writeString(bufferedWriter, "\n")
+				if errorWriting != nil {
+					return errorWriting
+				}
 			}
 		case "pv", "vp":
 			// put it into a node and print that.
@@ -106,17 +166,20 @@ func printResults(matchingNodes []*yqlib.NodeContext, cmd *cobra.Command) error
 			parentNode.Content = make([]*yaml.Node, 2)
 			parentNode.Content[0] = &yaml.Node{Kind: yaml.ScalarNode, Value: lib.PathStackToString(mappedDoc.PathStack)}
 			parentNode.Content[1] = mappedDoc.Node
-			if err := printValue(&parentNode, cmd); err != nil {
+			if err := printValue(&parentNode, bufferedWriter); err != nil {
 				return err
 			}
 		default:
-			if err := printValue(mappedDoc.Node, cmd); err != nil {
+			if err := printValue(mappedDoc.Node, bufferedWriter); err != nil {
 				return err
 			}
 			// Printing our Scalars does not print a new line at the end
 			// we only want to do that if there are more values (so users can easily script extraction of values in the yaml)
 			if index < len(matchingNodes)-1 && mappedDoc.Node.Kind == yaml.ScalarNode {
-				cmd.Print("\n")
+				errorWriting = writeString(bufferedWriter, "\n")
+				if errorWriting != nil {
+					return errorWriting
+				}
 			}
 		}
 	}
@@ -154,7 +217,12 @@ func mapYamlDecoder(updateData updateDataFn, encoder yqlib.Encoder) yamlDecoderF
 			log.Debugf("Read doc %v", currentIndex)
 			errorReading = decoder.Decode(&dataBucket)
 
-			if errorReading == io.EOF {
+			if errorReading == io.EOF && docIndexInt == 0 && currentIndex == 0 {
+				//empty document, lets just make one
+				child := yaml.Node{Kind: yaml.MappingNode}
+				dataBucket = yaml.Node{Kind: yaml.DocumentNode, Content: make([]*yaml.Node, 1)}
+				dataBucket.Content[0] = &child
+			} else if errorReading == io.EOF {
 				if !updateAll && currentIndex <= docIndexInt {
 					return fmt.Errorf("asked to process document index %v but there are only %v document(s)", docIndex, currentIndex)
 				}
@@ -167,6 +235,10 @@ func mapYamlDecoder(updateData updateDataFn, encoder yqlib.Encoder) yamlDecoderF
 				return errors.Wrapf(errorUpdating, "Error updating document at index %v", currentIndex)
 			}
 
+			if prettyPrint {
+				updateStyleOfNode(&dataBucket, 0)
+			}
+
 			errorWriting = encoder.Encode(&dataBucket)
 
 			if errorWriting != nil {
@@ -251,9 +323,9 @@ func readAndUpdate(stdOut io.Writer, inputFile string, updateData updateDataFn)
 
 	var encoder yqlib.Encoder
 	if outputToJSON {
-		encoder = yqlib.NewJsonEncoder(bufferedWriter)
+		encoder = yqlib.NewJsonEncoder(bufferedWriter, prettyPrint, indent)
 	} else {
-		encoder = yqlib.NewYamlEncoder(bufferedWriter)
+		encoder = yqlib.NewYamlEncoder(bufferedWriter, indent)
 	}
 	return readStream(inputFile, mapYamlDecoder(updateData, encoder))
 }

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "3.0.1"
+	Version = "3.1.0"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

examples/data2.yaml

@@ -4,4 +4,4 @@ c:
   toast: leave
   test: 1
   tell: 1
-  taco: cool
+  tasty.taco: cool

examples/data3.yaml

@@ -1,14 +1,4 @@
-deep1:
-   hostA:
-      value: 1234
-   notRelevant:
-      value: bananas
-   hostB:
-      value: 5678
-deep2:
-   hostC:
-      value: 1234
-   notRelevant:
-      value: bananas
-   hostD:
-      value: 5678
+a: "simple" # just the best
+b: [1, 3]
+c:
+  test: 1

examples/simple-anchor.yaml

@@ -1,4 +1,5 @@
 foo: &foo
   a: 1
 
-foobar: *foo
+foobar: 
+  <<: *foo

go.mod

@@ -1,6 +1,7 @@
 module github.com/mikefarah/yq/v3
 
 require (
+	github.com/kylelemons/godebug v1.1.0
 	github.com/pkg/errors v0.8.1
 	github.com/spf13/cobra v0.0.5
 	golang.org/x/tools v0.0.0-20191213221258-04c2e8eff935 // indirect

go.sum

@@ -9,6 +9,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mikefarah/yaml v2.1.0+incompatible h1:nu2cqmzk4WlWJNgnevY88faMcdrDzYGcsUjYFxEpB7Y=
 github.com/mikefarah/yaml/v2 v2.4.0 h1:eYqfooY0BnvKTJxr7+ABJs13n3dg9n347GScDaU2Lww=

pkg/yqlib/data_navigator.go

@@ -31,11 +31,19 @@ func (n *navigator) Traverse(value *yaml.Node, path []string) error {
 }
 
 func (n *navigator) doTraverse(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
+
+	if value.Kind == yaml.ScalarNode {
+		return n.navigationStrategy.Visit(NewNodeContext(value, head, tail, pathStack))
+	}
+
 	log.Debug("head %v", head)
 	DebugNode(value)
+
 	var errorDeepSplatting error
-	if head == "**" && value.Kind != yaml.ScalarNode {
-		errorDeepSplatting = n.recurse(value, head, tail, pathStack)
+	if head == "**" {
+		if len(pathStack) == 0 || pathStack[len(pathStack)-1] != "<<" {
+			errorDeepSplatting = n.recurse(value, head, tail, pathStack)
+		}
 		// ignore errors here, we are deep splatting so we may accidently give a string key
 		// to an array sequence
 		if len(tail) > 0 {
@@ -61,7 +69,7 @@ func (n *navigator) getOrReplace(original *yaml.Node, expectedKind yaml.Kind) *y
 }
 
 func (n *navigator) recurse(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
-	log.Debug("recursing, processing %v", head)
+	log.Debug("recursing, processing %v, pathStack %v", head, pathStackToString(pathStack))
 	switch value.Kind {
 	case yaml.MappingNode:
 		log.Debug("its a map with %v entries", len(value.Content)/2)
@@ -81,8 +89,14 @@ func (n *navigator) recurse(value *yaml.Node, head string, tail []string, pathSt
 		log.Debug("its an alias!")
 		DebugNode(value.Alias)
 		if n.navigationStrategy.FollowAlias(NewNodeContext(value, head, tail, pathStack)) {
-			log.Debug("following the alias")
-			return n.recurse(value.Alias, head, tail, pathStack)
+
+			if value.Alias.Kind == yaml.ScalarNode {
+				log.Debug("alias to a scalar")
+				return n.navigationStrategy.Visit(NewNodeContext(value.Alias, head, tail, pathStack))
+			} else {
+				log.Debug("following the alias")
+				return n.recurse(value.Alias, head, tail, pathStack)
+			}
 		}
 		return nil
 	default:
@@ -93,11 +107,9 @@ func (n *navigator) recurse(value *yaml.Node, head string, tail []string, pathSt
 func (n *navigator) recurseMap(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
 	traversedEntry := false
 	errorVisiting := n.visitMatchingEntries(value, head, tail, pathStack, func(contents []*yaml.Node, indexInMap int) error {
-		log.Debug("recurseMap: visitMatchingEntries")
+		log.Debug("recurseMap: visitMatchingEntries for %v", contents[indexInMap].Value)
 		n.navigationStrategy.DebugVisitedNodes()
 		newPathStack := append(pathStack, contents[indexInMap].Value)
-		log.Debug("appended %v", contents[indexInMap].Value)
-		n.navigationStrategy.DebugVisitedNodes()
 		log.Debug("should I traverse? head: %v, path: %v", head, pathStackToString(newPathStack))
 		DebugNode(value)
 		if n.navigationStrategy.ShouldTraverse(NewNodeContext(contents[indexInMap+1], head, tail, newPathStack), contents[indexInMap].Value) {
@@ -171,10 +183,10 @@ func (n *navigator) visitAliases(contents []*yaml.Node, head string, tail []stri
 	// a node can either be
 	// an alias to one other node (e.g. <<: *blah)
 	// or a sequence of aliases   (e.g. <<: [*blah, *foo])
-	log.Debug("checking for aliases")
+	log.Debug("checking for aliases, head: %v, pathstack: %v", head, pathStackToString(pathStack))
 	for index := len(contents) - 2; index >= 0; index = index - 2 {
 
-		if contents[index+1].Kind == yaml.AliasNode {
+		if contents[index+1].Kind == yaml.AliasNode && contents[index].Value == "<<" {
 			valueNode := contents[index+1]
 			log.Debug("found an alias")
 			DebugNode(contents[index])

pkg/yqlib/delete_navigation_strategy.go

@@ -15,7 +15,7 @@ func DeleteNavigationStrategy(pathElementToDelete string) NavigationStrategy {
 			return false
 		},
 		autoCreateMap: func(nodeContext NodeContext) bool {
-			return true
+			return false
 		},
 		visit: func(nodeContext NodeContext) error {
 			node := nodeContext.Node

pkg/yqlib/encoder.go

@@ -15,9 +15,12 @@ type yamlEncoder struct {
 	encoder *yaml.Encoder
 }
 
-func NewYamlEncoder(destination io.Writer) Encoder {
+func NewYamlEncoder(destination io.Writer, indent int) Encoder {
 	var encoder = yaml.NewEncoder(destination)
-	encoder.SetIndent(2)
+	if indent < 0 {
+		indent = 0
+	}
+	encoder.SetIndent(indent)
 	return &yamlEncoder{encoder}
 }
 
@@ -29,8 +32,16 @@ type jsonEncoder struct {
 	encoder *json.Encoder
 }
 
-func NewJsonEncoder(destination io.Writer) Encoder {
+func NewJsonEncoder(destination io.Writer, prettyPrint bool, indent int) Encoder {
 	var encoder = json.NewEncoder(destination)
+	var indentString = ""
+
+	for index := 0; index < indent; index++ {
+		indentString = indentString + " "
+	}
+	if prettyPrint {
+		encoder.SetIndent("", indentString)
+	}
 	return &jsonEncoder{encoder}
 }
 

pkg/yqlib/lib.go

@@ -51,7 +51,15 @@ func mergePathStackToString(pathStack []interface{}, appendArrays bool) string {
 			}
 
 		default:
-			sb.WriteString(fmt.Sprintf("%v", path))
+			s := fmt.Sprintf("%v", path)
+			hasDot := strings.Contains(s, ".")
+			if hasDot {
+				sb.WriteString("[")
+			}
+			sb.WriteString(s)
+			if hasDot {
+				sb.WriteString("]")
+			}
 		}
 
 		if index < len(pathStack)-1 {

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '3.0.1'
+version: '3.1.0'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.