Beta

.gitignore

@@ -23,6 +23,7 @@ _cgo_export.*
 
 _testmain.go
 coverage.out
+coverage.html
 *.exe
 *.test
 *.prof

README.md

@@ -7,21 +7,7 @@ a lightweight and portable command-line YAML processor
 
 The aim of the project is to be the [jq](https://github.com/stedolan/jq) or sed of yaml files.
 
-
-## Major upgrade - V3 beta is out! 
-
-This addresses a number of features requests and issues that have been raised :)
-
-Currently only available only available as a [binary release here](https://github.com/mikefarah/yq/releases/tag/3.0.0-beta) or via docker mikefarah/yq:3.0.0-beta!
-
-It does have a few breaking changes listed on the [release page](https://github.com/mikefarah/yq/releases/tag/3.0.0-beta)
-
-Looking forward to feedback - once this is out of beta it will be added to the remaining package managers, and be the default version downloaded (and merged into master).
-
-V2 will no longer have any new features added, and will be moved to a branch (v2). It will have limited maintenance for bugs for a few months.
-
 ## Install
-
 ### On MacOS:
 ```
 brew install yq
@@ -35,16 +21,16 @@ snap install yq
 `yq` installs with with [_strict confinement_](https://docs.snapcraft.io/snap-confinement/6233) in snap, this means it doesn't have direct access to root files. To read root files you can:
 
 ```
-sudo cat /etc/myfile | yq r - a.path
+sudo cat /etc/myfile | yq -r - somecommand
 ```
 
 And to write to a root file you can either use [sponge](https://linux.die.net/man/1/sponge):
 ```
-sudo cat /etc/myfile | yq w - a.path value | sudo sponge /etc/myfile
+sudo cat /etc/myfile | yq -r - somecommand | sudo sponge /etc/myfile
 ```
 or write to a temporary file:
 ```
-sudo cat /etc/myfile | yq w - a.path value | sudo tee /etc/myfile.tmp
+sudo cat /etc/myfile | yq -r - somecommand | sudo tee /etc/myfile.tmp
 sudo mv /etc/myfile.tmp /etc/myfile
 rm /etc/myfile.tmp
 ```
@@ -129,9 +115,6 @@ Use "yq [command] --help" for more information about a command.
 ```
 
 ## Contribute
-
-**Note: v3 is currently in progress - for the moment I won't be accepting new feature PRs until v3 is ready :)**
-
 1. `scripts/devtools.sh`
 2. `make [local] vendor`
 3. add unit tests

Upgrade Notes

@@ -0,0 +1,56 @@
+
+# New Features
+  - Keeps yaml comments and formatting (string blocks are saved, number formatting is preserved, so it won't drop off trailing 0s for values like 0.10, which is important when that's a version entry )
+  
+  - Handles anchors! (doc link)
+  - Can specify yaml tags (e.g. !!int), quoting values no longer sufficient, need to specify the tag value instead.
+  - Can print out matching paths and values when splatting (doc link)
+  - JSON output works for all commands! Yaml files with multiple documents are printed out as one JSON document per line. 
+  - Deep splat (**) to match arbitrary paths, (doc link)
+
+
+# Breaking changes
+
+## Update scripts file format has changed to be more powerful. 
+Comments can be added, and delete commands have been introduced.
+
+## Reading and splatting, matching results are printed once per line.
+  e.g:
+
+```json
+parent:
+  childA: 
+    no: matches here
+  childB:
+    there: matches
+    hi: no match
+    there2: also matches
+```
+
+yq r sample.yaml 'parent.*.there*'
+
+old
+```yaml
+- null
+- - matches
+  - also matches
+```
+
+new
+```yaml
+matches
+also matches
+```
+
+and you can print the matching paths:
+
+yq r --printMode pv sample.yaml 'parent.*.there*'
+
+```yaml
+parent.childB.there: matches
+parent.childB.there2: also matches
+```
+
+# Merge command
+- New flag 'autocreates' missing entries in target by default, new flag to turn that off.
+

commands_test.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/mikefarah/yq/v2/test"
+	"github.com/mikefarah/yq/v3/test"
 	"github.com/spf13/cobra"
 )
 
@@ -63,30 +63,6 @@ func TestRootCmd_VerboseShort(t *testing.T) {
 	}
 }
 
-func TestRootCmd_TrimLong(t *testing.T) {
-	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "--trim")
-	if result.Error != nil {
-		t.Error(result.Error)
-	}
-
-	if !trimOutput {
-		t.Error("Expected trimOutput to be true")
-	}
-}
-
-func TestRootCmd_TrimShort(t *testing.T) {
-	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "-t")
-	if result.Error != nil {
-		t.Error(result.Error)
-	}
-
-	if !trimOutput {
-		t.Error("Expected trimOutput to be true")
-	}
-}
-
 func TestRootCmd_VersionShort(t *testing.T) {
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, "-V")
@@ -115,7 +91,140 @@ func TestReadCmd(t *testing.T) {
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	test.AssertResult(t, "2\n", result.Output)
+	test.AssertResult(t, "2", result.Output)
+}
+
+func TestReadWithKeyAndValueCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -p pv examples/sample.yaml b.c")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	test.AssertResult(t, "b.c: 2\n", result.Output)
+}
+
+func TestReadArrayCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -p pv examples/sample.yaml b.e.1.name")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	test.AssertResult(t, "b.e.1.name: sam\n", result.Output)
+}
+
+func TestReadDeepSplatCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -p pv examples/sample.yaml b.**")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `b.c: 2
+b.d.[0]: 3
+b.d.[1]: 4
+b.d.[2]: 5
+b.e.[0].name: fred
+b.e.[0].value: 3
+b.e.[1].name: sam
+b.e.[1].value: 4
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestReadDeepSplatWithSuffixCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -p pv examples/sample.yaml b.**.name")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `b.e.[0].name: fred
+b.e.[1].name: sam
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestReadWithKeyCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -p p examples/sample.yaml b.c")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	test.AssertResult(t, "b.c", result.Output)
+}
+
+func TestReadAnchorsCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read examples/simple-anchor.yaml foobar.a")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	test.AssertResult(t, "1", result.Output)
+}
+
+func TestReadAnchorsWithKeyAndValueCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -p pv examples/simple-anchor.yaml foobar.a")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	test.AssertResult(t, "foobar.a: 1\n", result.Output)
+}
+
+func TestReadMergeAnchorsOriginalCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read examples/merge-anchor.yaml foobar.a")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	test.AssertResult(t, "original", result.Output)
+}
+
+func TestReadMergeAnchorsOverrideCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read examples/merge-anchor.yaml foobar.thing")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	test.AssertResult(t, "ice", result.Output)
+}
+
+func TestReadMergeAnchorsPrefixMatchCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "r -p pv examples/merge-anchor.yaml foobar.th*")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `foobar.thirty: well beyond
+foobar.thing: ice
+foobar.thirsty: yep
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestReadMergeAnchorsListOriginalCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read examples/merge-anchor.yaml foobarList.a")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	test.AssertResult(t, "original", result.Output)
+}
+
+func TestReadMergeAnchorsListOverrideInListCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read examples/merge-anchor.yaml foobarList.thing")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	test.AssertResult(t, "coconut", result.Output)
+}
+
+func TestReadMergeAnchorsListOverrideCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read examples/merge-anchor.yaml foobarList.c")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	test.AssertResult(t, "newbar", result.Output)
 }
 
 func TestReadInvalidDocumentIndexCmd(t *testing.T) {
@@ -134,7 +243,7 @@ func TestReadBadDocumentIndexCmd(t *testing.T) {
 	if result.Error == nil {
 		t.Error("Expected command to fail due to invalid path")
 	}
-	expectedOutput := `asked to process document index 1 but there are only 1 document(s)`
+	expectedOutput := `Could not process document index 1 as there are only 1 document(s)`
 	test.AssertResult(t, expectedOutput, result.Error.Error())
 }
 
@@ -157,7 +266,16 @@ func TestReadMultiCmd(t *testing.T) {
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	test.AssertResult(t, "here\n", result.Output)
+	test.AssertResult(t, "here", result.Output)
+}
+
+func TestReadMultiWithKeyAndValueCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -p vp -d 1 examples/multiple_docs.yaml another.document")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	test.AssertResult(t, "another.document: here\n", result.Output)
 }
 
 func TestReadMultiAllCmd(t *testing.T) {
@@ -167,9 +285,21 @@ func TestReadMultiAllCmd(t *testing.T) {
 		t.Error(result.Error)
 	}
 	test.AssertResult(t,
-		`- first document
-- second document
-- third document
+		`first document
+second document
+third document`, result.Output)
+}
+
+func TestReadMultiAllWithKeyAndValueCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -p pv -d* examples/multiple_docs.yaml commonKey")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	test.AssertResult(t,
+		`commonKey: first document
+commonKey: second document
+commonKey: third document
 `, result.Output)
 }
 
@@ -179,7 +309,7 @@ func TestReadCmd_ArrayYaml(t *testing.T) {
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	test.AssertResult(t, "false\n", result.Output)
+	test.AssertResult(t, "false", result.Output)
 }
 
 func TestReadCmd_ArrayYaml_NoPath(t *testing.T) {
@@ -191,11 +321,13 @@ func TestReadCmd_ArrayYaml_NoPath(t *testing.T) {
 	expectedOutput := `- become: true
   gather_facts: false
   hosts: lalaland
-  name: Apply smth
+  name: "Apply smth"
   roles:
   - lala
   - land
   serial: 1
+- become: false
+  gather_facts: true
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -209,7 +341,7 @@ func TestReadCmd_ArrayYaml_OneElement(t *testing.T) {
 	expectedOutput := `become: true
 gather_facts: false
 hosts: lalaland
-name: Apply smth
+name: "Apply smth"
 roles:
 - lala
 - land
@@ -218,31 +350,67 @@ serial: 1
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
-func TestReadCmd_ArrayYaml_Splat(t *testing.T) {
+func TestReadCmd_ArrayYaml_SplatCmd(t *testing.T) {
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, "read examples/array.yaml [*]")
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `- become: true
+	expectedOutput := `become: true
+gather_facts: false
+hosts: lalaland
+name: "Apply smth"
+roles:
+- lala
+- land
+serial: 1
+become: false
+gather_facts: true
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestReadCmd_ArrayYaml_SplatWithKeyAndValueCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -p pv examples/array.yaml [*]")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `'[0]':
+  become: true
   gather_facts: false
   hosts: lalaland
-  name: Apply smth
+  name: "Apply smth"
   roles:
   - lala
   - land
   serial: 1
+'[1]':
+  become: false
+  gather_facts: true
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
+func TestReadCmd_ArrayYaml_SplatWithKeyCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "read -p p examples/array.yaml [*]")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `[0]
+[1]`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestReadCmd_ArrayYaml_SplatKey(t *testing.T) {
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, "read examples/array.yaml [*].gather_facts")
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := "- false\n"
+	expectedOutput := `false
+true`
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
@@ -250,9 +418,9 @@ func TestReadCmd_ArrayYaml_ErrorBadPath(t *testing.T) {
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, "read examples/array.yaml [x].gather_facts")
 	if result.Error == nil {
-		t.Error("Expected command to fail due to invalid path")
+		t.Error("Expected command to fail due to missing arg")
 	}
-	expectedOutput := `Error reading path in document index 0: error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
+	expectedOutput := `Error reading path in document index 0: Error parsing array index 'x' for '': strconv.ParseInt: parsing "x": invalid syntax`
 	test.AssertResult(t, expectedOutput, result.Error.Error())
 }
 
@@ -260,9 +428,9 @@ func TestReadCmd_ArrayYaml_Splat_ErrorBadPath(t *testing.T) {
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, "read examples/array.yaml [*].roles[x]")
 	if result.Error == nil {
-		t.Error("Expected command to fail due to invalid path")
+		t.Error("Expected command to fail due to missing arg")
 	}
-	expectedOutput := `Error reading path in document index 0: error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
+	expectedOutput := `Error reading path in document index 0: Error parsing array index 'x' for '[0].roles': strconv.ParseInt: parsing "x": invalid syntax`
 	test.AssertResult(t, expectedOutput, result.Error.Error())
 }
 
@@ -316,47 +484,117 @@ func TestReadCmd_ErrorBadPath(t *testing.T) {
 
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, fmt.Sprintf("read %s b.d.*.[x]", filename))
-	if result.Error == nil {
-		t.Fatal("Expected command to fail due to invalid path")
-	}
-	expectedOutput := `Error reading path in document index 0: error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
+	expectedOutput := `Error reading path in document index 0: Error parsing array index 'x' for 'b.d.e': strconv.ParseInt: parsing "x": invalid syntax`
 	test.AssertResult(t, expectedOutput, result.Error.Error())
 }
 
 func TestReadCmd_Verbose(t *testing.T) {
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "-v read examples/sample.yaml b.c")
+	result := test.RunCmd(cmd, "read -v examples/sample.yaml b.c")
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	test.AssertResult(t, "2\n", result.Output)
+	test.AssertResult(t, "2", result.Output)
 }
 
-func TestReadCmd_NoTrim(t *testing.T) {
+// func TestReadCmd_ToJson(t *testing.T) {
+// 	cmd := getRootCommand()
+// 	result := test.RunCmd(cmd, "read -j examples/sample.yaml b.c")
+// 	if result.Error != nil {
+// 		t.Error(result.Error)
+// 	}
+// 	test.AssertResult(t, "2\n", result.Output)
+// }
+
+// func TestReadCmd_ToJsonLong(t *testing.T) {
+// 	cmd := getRootCommand()
+// 	result := test.RunCmd(cmd, "read --tojson examples/sample.yaml b.c")
+// 	if result.Error != nil {
+// 		t.Error(result.Error)
+// 	}
+// 	test.AssertResult(t, "2\n", result.Output)
+// }
+
+func TestReadSplatPrefixCmd(t *testing.T) {
+	content := `a: 2
+b:
+ hi:
+   c: things
+   d: something else
+ there:
+   c: more things
+   d: more something else
+ there2:
+   c: more things also
+   d: more something else also
+`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "--trim=false read examples/sample.yaml b.c")
+	result := test.RunCmd(cmd, fmt.Sprintf("read %s b.there*.c", filename))
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	test.AssertResult(t, "2\n\n", result.Output)
+
+	expectedOutput := `more things
+more things also`
+	test.AssertResult(t, expectedOutput, result.Output)
 }
 
-func TestReadCmd_ToJson(t *testing.T) {
+func TestReadSplatPrefixWithKeyAndValueCmd(t *testing.T) {
+	content := `a: 2
+b:
+ hi:
+   c: things
+   d: something else
+ there:
+   c: more things
+   d: more something else
+ there2:
+   c: more things also
+   d: more something else also
+`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "read -j examples/sample.yaml b.c")
+	result := test.RunCmd(cmd, fmt.Sprintf("read -p pv %s b.there*.c", filename))
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	test.AssertResult(t, "2\n", result.Output)
+
+	expectedOutput := `b.there.c: more things
+b.there2.c: more things also
+`
+	test.AssertResult(t, expectedOutput, result.Output)
 }
 
-func TestReadCmd_ToJsonLong(t *testing.T) {
+func TestReadSplatPrefixWithKeyCmd(t *testing.T) {
+	content := `a: 2
+b:
+ hi:
+   c: things
+   d: something else
+ there:
+   c: more things
+   d: more something else
+ there2:
+   c: more things also
+   d: more something else also
+`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "read --tojson examples/sample.yaml b.c")
+	result := test.RunCmd(cmd, fmt.Sprintf("read -p p %s b.there*.c", filename))
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	test.AssertResult(t, "2\n", result.Output)
+
+	expectedOutput := `b.there.c
+b.there2.c`
+	test.AssertResult(t, expectedOutput, result.Output)
 }
 
 func TestPrefixCmd(t *testing.T) {
@@ -527,7 +765,7 @@ func TestPrefixCmd_Verbose(t *testing.T) {
 	defer test.RemoveTempYamlFile(filename)
 
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, fmt.Sprintf("-v prefix %s x", filename))
+	result := test.RunCmd(cmd, fmt.Sprintf("prefix %s x", filename))
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
@@ -569,6 +807,18 @@ func TestNewCmd(t *testing.T) {
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
+func TestNewArrayCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "new b[0] 3")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `b:
+- 3
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestNewCmd_Error(t *testing.T) {
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, "new b.c")
@@ -579,18 +829,6 @@ func TestNewCmd_Error(t *testing.T) {
 	test.AssertResult(t, expectedOutput, result.Error.Error())
 }
 
-func TestNewCmd_Verbose(t *testing.T) {
-	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "-v new b.c 3")
-	if result.Error != nil {
-		t.Error(result.Error)
-	}
-	expectedOutput := `b:
-  c: 3
-`
-	test.AssertResult(t, expectedOutput, result.Output)
-}
-
 func TestWriteCmd(t *testing.T) {
 	content := `b:
   c: 3
@@ -609,6 +847,52 @@ func TestWriteCmd(t *testing.T) {
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
+func TestWriteCmdScript(t *testing.T) {
+	content := `b:
+  c: 3
+`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	updateScript := `- command: update 
+  path: b.c
+  value: 7`
+	scriptFilename := test.WriteTempYamlFile(updateScript)
+	defer test.RemoveTempYamlFile(scriptFilename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("write --script %s %s", scriptFilename, filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `b:
+  c: 7
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestWriteCmdEmptyScript(t *testing.T) {
+	content := `b:
+  c: 3
+`
+	filename := test.WriteTempYamlFile(content)
+	defer test.RemoveTempYamlFile(filename)
+
+	updateScript := ``
+	scriptFilename := test.WriteTempYamlFile(updateScript)
+	defer test.RemoveTempYamlFile(scriptFilename)
+
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, fmt.Sprintf("write --script %s %s", scriptFilename, filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `b:
+  c: 3
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
 func TestWriteMultiCmd(t *testing.T) {
 	content := `b:
   c: 3
@@ -724,24 +1008,6 @@ func TestWriteCmd_ErrorUnreadableFile(t *testing.T) {
 	test.AssertResult(t, expectedOutput, result.Error.Error())
 }
 
-func TestWriteCmd_Verbose(t *testing.T) {
-	content := `b:
-  c: 3
-`
-	filename := test.WriteTempYamlFile(content)
-	defer test.RemoveTempYamlFile(filename)
-
-	cmd := getRootCommand()
-	result := test.RunCmd(cmd, fmt.Sprintf("-v write %s b.c 7", filename))
-	if result.Error != nil {
-		t.Error(result.Error)
-	}
-	expectedOutput := `b:
-  c: 7
-`
-	test.AssertResult(t, expectedOutput, result.Output)
-}
-
 func TestWriteCmd_Inplace(t *testing.T) {
 	content := `b:
   c: 3
@@ -786,7 +1052,7 @@ func TestWriteCmd_AppendEmptyArray(t *testing.T) {
 	defer test.RemoveTempYamlFile(filename)
 
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, fmt.Sprintf("write -v %s b[+] v", filename))
+	result := test.RunCmd(cmd, fmt.Sprintf("write %s b[+] v", filename))
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
@@ -806,7 +1072,7 @@ func TestWriteCmd_SplatArray(t *testing.T) {
 	defer test.RemoveTempYamlFile(filename)
 
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, fmt.Sprintf("write -v %s b[*].c new", filename))
+	result := test.RunCmd(cmd, fmt.Sprintf("write %s b[*].c new", filename))
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
@@ -826,7 +1092,7 @@ func TestWriteCmd_SplatMap(t *testing.T) {
 	defer test.RemoveTempYamlFile(filename)
 
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, fmt.Sprintf("write -v %s b.* new", filename))
+	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.* new", filename))
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
@@ -846,7 +1112,7 @@ func TestWriteCmd_SplatMapEmpty(t *testing.T) {
 	defer test.RemoveTempYamlFile(filename)
 
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, fmt.Sprintf("write -v %s b.c.* new", filename))
+	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.c.* new", filename))
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
@@ -857,7 +1123,7 @@ func TestWriteCmd_SplatMapEmpty(t *testing.T) {
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
-func TestDeleteYaml(t *testing.T) {
+func TestDeleteYamlCmd(t *testing.T) {
 	content := `a: 2
 b:
   c: things
@@ -880,35 +1146,28 @@ b:
 }
 
 func TestDeleteSplatYaml(t *testing.T) {
-	content := `a: 2
-b:
- hi:
-   c: things
-   d: something else
- hello:
-   c: things2
-   d: something else2
- there:
-   c: more things
-   d: more something else
+	content := `a: other
+b: [3, 4]
+c:
+  toast: leave
+  test: 1
+  tell: 1
+  taco: cool
 `
 	filename := test.WriteTempYamlFile(content)
 	defer test.RemoveTempYamlFile(filename)
 
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, fmt.Sprintf("delete -v %s b.*.c", filename))
+	result := test.RunCmd(cmd, fmt.Sprintf("delete %s c.te*", filename))
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
 
-	expectedOutput := `a: 2
-b:
-  hi:
-    d: something else
-  hello:
-    d: something else2
-  there:
-    d: more something else
+	expectedOutput := `a: other
+b: [3, 4]
+c:
+  toast: leave
+  taco: cool
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -926,7 +1185,7 @@ b:
 	defer test.RemoveTempYamlFile(filename)
 
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, fmt.Sprintf("delete -v %s b.hi[*].thing", filename))
+	result := test.RunCmd(cmd, fmt.Sprintf("delete %s b.hi[*].thing", filename))
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
@@ -957,7 +1216,7 @@ b:
 	defer test.RemoveTempYamlFile(filename)
 
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, fmt.Sprintf("delete -v %s b.there*.c", filename))
+	result := test.RunCmd(cmd, fmt.Sprintf("delete %s b.there*.c", filename))
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
@@ -1048,10 +1307,25 @@ func TestMergeCmd(t *testing.T) {
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `a: simple
-b:
-- 1
-- 2
+	expectedOutput := `a: simple # just the best
+b: [1, 2]
+c:
+  test: 1
+  toast: leave
+  tell: 1
+  taco: cool
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeNoAutoCreateCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "merge -c=false examples/data1.yaml examples/data2.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `a: simple # just the best
+b: [1, 2]
 c:
   test: 1
 `
@@ -1060,14 +1334,12 @@ c:
 
 func TestMergeOverwriteCmd(t *testing.T) {
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "merge --overwrite examples/data1.yaml examples/data2.yaml")
+	result := test.RunCmd(cmd, "merge -c=false --overwrite examples/data1.yaml examples/data2.yaml")
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `a: other
-b:
-- 3
-- 4
+	expectedOutput := `a: other # better than the original
+b: [3, 4]
 c:
   test: 1
 `
@@ -1076,16 +1348,12 @@ c:
 
 func TestMergeAppendCmd(t *testing.T) {
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "merge --append examples/data1.yaml examples/data2.yaml")
+	result := test.RunCmd(cmd, "merge --autocreate=false --append examples/data1.yaml examples/data2.yaml")
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `a: simple
-b:
-- 1
-- 2
-- 3
-- 4
+	expectedOutput := `a: simple # just the best
+b: [1, 2, 3, 4]
 c:
   test: 1
 `
@@ -1094,16 +1362,12 @@ c:
 
 func TestMergeOverwriteAndAppendCmd(t *testing.T) {
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "merge --append --overwrite examples/data1.yaml examples/data2.yaml")
+	result := test.RunCmd(cmd, "merge --autocreate=false --append --overwrite examples/data1.yaml examples/data2.yaml")
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `a: other
-b:
-- 1
-- 2
-- 3
-- 4
+	expectedOutput := `a: other # better than the original
+b: [1, 2, 3, 4]
 c:
   test: 1
 `
@@ -1116,35 +1380,32 @@ func TestMergeArraysCmd(t *testing.T) {
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `- 1
-- 2
-- 3
-- 4
-- 5
+	expectedOutput := `[1, 2, 3, 4, 5]
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
 func TestMergeCmd_Multi(t *testing.T) {
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "merge -d1 examples/multiple_docs_small.yaml examples/data2.yaml")
+	result := test.RunCmd(cmd, "merge -d1 examples/multiple_docs_small.yaml examples/data1.yaml")
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
 	expectedOutput := `a: Easy! as one two three
 ---
-a: other
 another:
   document: here
+a: simple # just the best
 b:
-- 3
-- 4
+- 1
+- 2
 c:
   test: 1
 ---
 - 1
-- 2`
-	test.AssertResult(t, expectedOutput, strings.Trim(result.Output, "\n "))
+- 2
+`
+	test.AssertResult(t, expectedOutput, result.Output)
 }
 
 func TestMergeYamlMultiAllCmd(t *testing.T) {
@@ -1166,14 +1427,15 @@ something: good`
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `apples: green
-b:
+	expectedOutput := `b:
   c: 3
+apples: green
 something: good
 ---
+something: else
 apples: red
-something: else`
-	test.AssertResult(t, expectedOutput, strings.Trim(result.Output, "\n "))
+`
+	test.AssertResult(t, expectedOutput, result.Output)
 }
 
 func TestMergeYamlMultiAllOverwriteCmd(t *testing.T) {
@@ -1195,14 +1457,15 @@ something: good`
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `apples: red
-b:
+	expectedOutput := `b:
   c: 3
+apples: red
 something: good
 ---
+something: good
 apples: red
-something: good`
-	test.AssertResult(t, expectedOutput, strings.Trim(result.Output, "\n "))
+`
+	test.AssertResult(t, expectedOutput, result.Output)
 }
 
 func TestMergeCmd_Error(t *testing.T) {
@@ -1223,29 +1486,13 @@ func TestMergeCmd_ErrorUnreadableFile(t *testing.T) {
 	}
 	var expectedOutput string
 	if runtime.GOOS == "windows" {
-		expectedOutput = `Error updating document at index 0: open fake-unknown: The system cannot find the file specified.`
+		expectedOutput = `open fake-unknown: The system cannot find the file specified.`
 	} else {
-		expectedOutput = `Error updating document at index 0: open fake-unknown: no such file or directory`
+		expectedOutput = `open fake-unknown: no such file or directory`
 	}
 	test.AssertResult(t, expectedOutput, result.Error.Error())
 }
 
-func TestMergeCmd_Verbose(t *testing.T) {
-	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "-v merge examples/data1.yaml examples/data2.yaml")
-	if result.Error != nil {
-		t.Error(result.Error)
-	}
-	expectedOutput := `a: simple
-b:
-- 1
-- 2
-c:
-  test: 1
-`
-	test.AssertResult(t, expectedOutput, result.Output)
-}
-
 func TestMergeCmd_Inplace(t *testing.T) {
 	filename := test.WriteTempYamlFile(test.ReadTempYamlFile("examples/data1.yaml"))
 	err := os.Chmod(filename, os.FileMode(int(0666)))
@@ -1261,13 +1508,15 @@ func TestMergeCmd_Inplace(t *testing.T) {
 	}
 	info, _ := os.Stat(filename)
 	gotOutput := test.ReadTempYamlFile(filename)
-	expectedOutput := `a: simple
-b:
-- 1
-- 2
+	expectedOutput := `a: simple # just the best
+b: [1, 2]
 c:
-  test: 1`
-	test.AssertResult(t, expectedOutput, strings.Trim(gotOutput, "\n "))
+  test: 1
+  toast: leave
+  tell: 1
+  taco: cool
+`
+	test.AssertResult(t, expectedOutput, gotOutput)
 	test.AssertResult(t, os.FileMode(int(0666)), info.Mode())
 }
 
@@ -1277,10 +1526,17 @@ func TestMergeAllowEmptyCmd(t *testing.T) {
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `a: simple
-b:
-- 1
-- 2
+	expectedOutput := `a: simple # just the best
+b: [1, 2]
+c:
+  test: 1
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
+
+func TestMergeDontAllowEmptyCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "merge examples/data1.yaml examples/empty.yaml")
+	expectedOutput := `Could not process document index 0 as there are only 0 document(s)`
+	test.AssertResult(t, expectedOutput, result.Error.Error())
+}

compare.sh

@@ -0,0 +1,13 @@
+GREEN='\033[0;32m'
+NC='\033[0m'
+
+echo "${GREEN}---Old---${NC}"
+yq $@ > /tmp/yq-old-output
+cat /tmp/yq-old-output
+
+echo "${GREEN}---New---${NC}"
+./yq $@ > /tmp/yq-new-output
+cat /tmp/yq-new-output
+
+echo "${GREEN}---Diff---${NC}"
+colordiff /tmp/yq-old-output /tmp/yq-new-output

examples/array.yaml

@@ -7,3 +7,5 @@
     - lala
     - land
   serial: 1
+- become: false
+  gather_facts: true

examples/data1.yaml

@@ -1,2 +1,4 @@
-a: simple
+a: simple # just the best
 b: [1, 2]
+c:
+  test: 1

examples/data2.yaml

@@ -1,4 +1,7 @@
-a: other
+a: other # better than the original
 b: [3, 4]
 c:
+  toast: leave
   test: 1
+  tell: 1
+  taco: cool

examples/instruction_sample.yaml

@@ -1,2 +1,10 @@
-b.c: cat
-b.e[+].name: Mike Farah
+- command: update 
+  path: b.c
+  value:
+    #great 
+    things: frog # wow!
+- command: update
+  path: b.e[+].name
+  value: Mike Farah
+- command: delete
+  path: b.d

examples/merge-anchor.yaml

@@ -0,0 +1,19 @@
+foo: &foo
+  a: original
+  thing: coolasdf
+  thirsty: yep
+
+bar: &bar
+  b: 2
+  thing: coconut
+  c: oldbar
+
+foobarList:
+  <<: [*foo,*bar]
+  c: newbar
+
+foobar:
+  <<: *foo
+  thirty: well beyond
+  thing: ice
+  c: 3

examples/sample.yaml

@@ -1,7 +1,7 @@
-a: Easy! as one two three
+a: true
 b:
   c: 2
-  d: [3, 4]
+  d: [3, 4, 5]
   e:
     - name: fred
       value: 3

examples/sample2.yaml

@@ -1,9 +1,2 @@
-a: Easy! as one two three
 b:
-  c: things
-  d: whatever
-things:
- thing1:
-  cat: 'fred'
- thing2:
-  cat: 'sam'
+  c: things

examples/sample_array_2.yaml

@@ -1 +1,2 @@
-[4,5]
+- 4
+- 5

examples/simple-anchor.yaml

@@ -0,0 +1,4 @@
+foo: &foo
+  a: 1
+
+foobar: *foo

go.mod

@@ -1,12 +1,13 @@
-module github.com/mikefarah/yq/v2
+module github.com/mikefarah/yq/v3
 
 require (
-	github.com/mikefarah/yaml/v2 v2.4.0
+	github.com/mikefarah/yaml/v2 v2.4.0 // indirect
 	github.com/pkg/errors v0.8.1
 	github.com/spf13/cobra v0.0.5
-	golang.org/x/tools v0.0.0-20191030203535-5e247c9ad0a0 // indirect
-	gopkg.in/imdario/mergo.v0 v0.3.7
+	golang.org/x/tools v0.0.0-20191213221258-04c2e8eff935 // indirect
+	gopkg.in/imdario/mergo.v0 v0.3.7 // indirect
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
+	gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2
 )
 
 go 1.13

go.sum

@@ -10,8 +10,11 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mikefarah/yaml v2.1.0+incompatible h1:nu2cqmzk4WlWJNgnevY88faMcdrDzYGcsUjYFxEpB7Y=
 github.com/mikefarah/yaml/v2 v2.4.0 h1:eYqfooY0BnvKTJxr7+ABJs13n3dg9n347GScDaU2Lww=
 github.com/mikefarah/yaml/v2 v2.4.0/go.mod h1:ahVqZF4n1W4NqwvVnZzC4es67xsW9uR/RRf2RRxieJU=
+github.com/mikefarah/yq v2.4.0+incompatible h1:oBxbWy8R9hI3BIUUxEf0CzikWa2AgnGrGhvGQt5jgjk=
+github.com/mikefarah/yq/v2 v2.4.1 h1:tajDonaFK6WqitSZExB6fKlWQy/yCkptqxh2AXEe3N4=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@@ -32,14 +35,21 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20191030203535-5e247c9ad0a0 h1:s5lp4ug7qHzUccgyFdjsX7OZDzHXRaePrF3B3vmUiuM=
 golang.org/x/tools v0.0.0-20191030203535-5e247c9ad0a0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191213221258-04c2e8eff935 h1:kJQZhwFzSwJS2BxboKjdZzWczQOZx8VuH7Y8hhuGUtM=
+golang.org/x/tools v0.0.0-20191213221258-04c2e8eff935/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/imdario/mergo.v0 v0.3.7 h1:QDotlIZtaO/p+Um0ok18HRTpq5i5/SAk/qprsor+9c8=
@@ -48,3 +58,5 @@ gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2 h1:XZx7nhd5GMaZpmDaEHFVafUZC7ya0fuo7cSJ3UCKYmM=
+gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/marshal/json_converter.go

@@ -1,54 +0,0 @@
-package marshal
-
-import (
-	"encoding/json"
-	"fmt"
-	"strconv"
-
-	yaml "github.com/mikefarah/yaml/v2"
-)
-
-type JsonConverter interface {
-	JsonToString(context interface{}) (string, error)
-}
-
-type jsonConverter struct{}
-
-func NewJsonConverter() JsonConverter {
-	return &jsonConverter{}
-}
-
-func (j *jsonConverter) JsonToString(context interface{}) (string, error) {
-	out, err := json.Marshal(j.toJSON(context))
-	if err != nil {
-		return "", fmt.Errorf("error printing yaml as json: %v", err)
-	}
-	return string(out), nil
-}
-
-func (j *jsonConverter) toJSON(context interface{}) interface{} {
-	switch context := context.(type) {
-	case []interface{}:
-		oldArray := context
-		newArray := make([]interface{}, len(oldArray))
-		for index, value := range oldArray {
-			newArray[index] = j.toJSON(value)
-		}
-		return newArray
-	case yaml.MapSlice:
-		oldMap := context
-		newMap := make(map[string]interface{})
-		for _, entry := range oldMap {
-			if str, ok := entry.Key.(string); ok {
-				newMap[str] = j.toJSON(entry.Value)
-			} else if i, ok := entry.Key.(int); ok {
-				newMap[strconv.Itoa(i)] = j.toJSON(entry.Value)
-			} else if b, ok := entry.Key.(bool); ok {
-				newMap[strconv.FormatBool(b)] = j.toJSON(entry.Value)
-			}
-		}
-		return newMap
-	default:
-		return context
-	}
-}

pkg/marshal/json_converter_test.go

@@ -1,48 +0,0 @@
-package marshal
-
-import (
-	"testing"
-
-	"github.com/mikefarah/yq/v2/test"
-)
-
-func TestJsonToString(t *testing.T) {
-	var data = test.ParseData(`
----
-b:
-  c: 2
-`)
-	got, _ := NewJsonConverter().JsonToString(data)
-	test.AssertResult(t, "{\"b\":{\"c\":2}}", got)
-}
-
-func TestJsonToString_withIntKey(t *testing.T) {
-	var data = test.ParseData(`
----
-b:
-  2: c
-`)
-	got, _ := NewJsonConverter().JsonToString(data)
-	test.AssertResult(t, `{"b":{"2":"c"}}`, got)
-}
-
-func TestJsonToString_withBoolKey(t *testing.T) {
-	var data = test.ParseData(`
----
-b:
-  false: c
-`)
-	got, _ := NewJsonConverter().JsonToString(data)
-	test.AssertResult(t, `{"b":{"false":"c"}}`, got)
-}
-
-func TestJsonToString_withArray(t *testing.T) {
-	var data = test.ParseData(`
----
-b:
-  - item: one
-  - item: two
-`)
-	got, _ := NewJsonConverter().JsonToString(data)
-	test.AssertResult(t, "{\"b\":[{\"item\":\"one\"},{\"item\":\"two\"}]}", got)
-}

pkg/marshal/yaml_converter.go

@@ -1,43 +0,0 @@
-package marshal
-
-import (
-	"strings"
-
-	yaml "github.com/mikefarah/yaml/v2"
-	errors "github.com/pkg/errors"
-)
-
-type YamlConverter interface {
-	YamlToString(context interface{}, trimOutput bool) (string, error)
-}
-
-type yamlConverter struct{}
-
-func NewYamlConverter() YamlConverter {
-	return &yamlConverter{}
-}
-
-func (y *yamlConverter) YamlToString(context interface{}, trimOutput bool) (string, error) {
-	switch context := context.(type) {
-	case string:
-		return context, nil
-	default:
-		return y.marshalContext(context, trimOutput)
-	}
-}
-
-func (y *yamlConverter) marshalContext(context interface{}, trimOutput bool) (string, error) {
-	out, err := yaml.Marshal(context)
-
-	if err != nil {
-		return "", errors.Wrap(err, "error printing yaml")
-	}
-
-	outStr := string(out)
-	// trim the trailing new line as it's easier for a script to add
-	// it in if required than to remove it
-	if trimOutput {
-		return strings.Trim(outStr, "\n "), nil
-	}
-	return outStr, nil
-}

pkg/marshal/yaml_converter_test.go

@@ -1,52 +0,0 @@
-package marshal
-
-import (
-	"testing"
-
-	"github.com/mikefarah/yq/v2/test"
-)
-
-func TestYamlToString(t *testing.T) {
-	var raw = `b:
-  c: 2
-`
-	var data = test.ParseData(raw)
-	got, _ := NewYamlConverter().YamlToString(data, false)
-	test.AssertResult(t, raw, got)
-}
-
-func TestYamlToString_withTrim(t *testing.T) {
-	var raw = `b:
-  c: 2`
-	var data = test.ParseData(raw)
-	got, _ := NewYamlConverter().YamlToString(data, true)
-	test.AssertResult(t, raw, got)
-}
-
-func TestYamlToString_withIntKey(t *testing.T) {
-	var raw = `b:
-  2: c
-`
-	var data = test.ParseData(raw)
-	got, _ := NewYamlConverter().YamlToString(data, false)
-	test.AssertResult(t, raw, got)
-}
-
-func TestYamlToString_withBoolKey(t *testing.T) {
-	var raw = `b:
-  false: c
-`
-	var data = test.ParseData(raw)
-	got, _ := NewYamlConverter().YamlToString(data, false)
-	test.AssertResult(t, raw, got)
-}
-
-func TestYamlToString_withArray(t *testing.T) {
-	var raw = `b:
-- item: one
-- item: two
-`
-	var data = test.ParseData(raw)
-	got, _ := NewYamlConverter().YamlToString(data, false)
-	test.AssertResult(t, raw, got)
-}

pkg/yqlib/data_navigator.go

@@ -1,376 +1,245 @@
 package yqlib
 
 import (
-	"fmt"
-	"reflect"
 	"strconv"
-	"strings"
 
-	yaml "github.com/mikefarah/yaml/v2"
-	logging "gopkg.in/op/go-logging.v1"
+	errors "github.com/pkg/errors"
+	yaml "gopkg.in/yaml.v3"
 )
 
 type DataNavigator interface {
-	ReadChildValue(child interface{}, remainingPaths []string) (interface{}, error)
-	UpdatedChildValue(child interface{}, remainingPaths []string, value interface{}) interface{}
-	DeleteChildValue(child interface{}, remainingPaths []string) (interface{}, error)
+	Traverse(value *yaml.Node, path []string) error
 }
 
 type navigator struct {
-	log *logging.Logger
+	navigationStrategy NavigationStrategy
 }
 
-func NewDataNavigator(l *logging.Logger) DataNavigator {
+func NewDataNavigator(NavigationStrategy NavigationStrategy) DataNavigator {
 	return &navigator{
-		log: l,
+		navigationStrategy: NavigationStrategy,
 	}
 }
 
-func (n *navigator) ReadChildValue(child interface{}, remainingPaths []string) (interface{}, error) {
-	if len(remainingPaths) == 0 {
-		return child, nil
+func (n *navigator) Traverse(value *yaml.Node, path []string) error {
+	realValue := value
+	emptyArray := make([]interface{}, 0)
+	if realValue.Kind == yaml.DocumentNode {
+		log.Debugf("its a document! returning the first child")
+		return n.doTraverse(value.Content[0], "", path, emptyArray)
 	}
-	return n.recurse(child, remainingPaths[0], remainingPaths[1:])
+	return n.doTraverse(value, "", path, emptyArray)
 }
 
-func (n *navigator) UpdatedChildValue(child interface{}, remainingPaths []string, value interface{}) interface{} {
-	if len(remainingPaths) == 0 {
-		return value
-	}
-	n.log.Debugf("UpdatedChildValue for child %v with path %v to set value %v", child, remainingPaths, value)
-	n.log.Debugf("type of child is %v", reflect.TypeOf(child))
-
-	switch child := child.(type) {
-	case nil:
-		if remainingPaths[0] == "+" || remainingPaths[0] == "*" {
-			return n.writeArray(child, remainingPaths, value)
-		}
-	case []interface{}:
-		_, nextIndexErr := strconv.ParseInt(remainingPaths[0], 10, 64)
-		arrayCommand := nextIndexErr == nil || remainingPaths[0] == "+" || remainingPaths[0] == "*"
-		if arrayCommand {
-			return n.writeArray(child, remainingPaths, value)
-		}
-	}
-	return n.writeMap(child, remainingPaths, value)
-}
-
-func (n *navigator) DeleteChildValue(child interface{}, remainingPaths []string) (interface{}, error) {
-	n.log.Debugf("DeleteChildValue for %v for %v\n", remainingPaths, child)
-	if len(remainingPaths) == 0 {
-		return child, nil
-	}
-	var head = remainingPaths[0]
-	var tail = remainingPaths[1:]
-	switch child := child.(type) {
-	case yaml.MapSlice:
-		return n.deleteMap(child, remainingPaths)
-	case []interface{}:
-		if head == "*" {
-			return n.deleteArraySplat(child, tail)
-		}
-		index, err := strconv.ParseInt(head, 10, 64)
-		if err != nil {
-			return nil, fmt.Errorf("error accessing array: %v", err)
-		}
-		return n.deleteArray(child, remainingPaths, index)
-	}
-	return child, nil
-}
-
-func (n *navigator) recurse(value interface{}, head string, tail []string) (interface{}, error) {
-	switch value := value.(type) {
-	case []interface{}:
-		if head == "*" {
-			return n.readArraySplat(value, tail)
-		}
-		index, err := strconv.ParseInt(head, 10, 64)
-		if err != nil {
-			return nil, fmt.Errorf("error accessing array: %v", err)
-		}
-		return n.readArray(value, index, tail)
-	case yaml.MapSlice:
-		return n.readMap(value, head, tail)
-	default:
-		return nil, nil
-	}
-}
-
-func (n *navigator) matchesKey(key string, actual interface{}) bool {
-	var actualString = fmt.Sprintf("%v", actual)
-	var prefixMatch = strings.TrimSuffix(key, "*")
-	if prefixMatch != key {
-		return strings.HasPrefix(actualString, prefixMatch)
-	}
-	return actualString == key
-}
-
-func (n *navigator) entriesInSlice(context yaml.MapSlice, key string) []*yaml.MapItem {
-	var matches = make([]*yaml.MapItem, 0)
-	for idx := range context {
-		var entry = &context[idx]
-		if n.matchesKey(key, entry.Key) {
-			matches = append(matches, entry)
-		}
-	}
-	return matches
-}
-
-func (n *navigator) getMapSlice(context interface{}) yaml.MapSlice {
-	var mapSlice yaml.MapSlice
-	switch context := context.(type) {
-	case yaml.MapSlice:
-		mapSlice = context
-	default:
-		mapSlice = make(yaml.MapSlice, 0)
-	}
-	return mapSlice
-}
-
-func (n *navigator) getArray(context interface{}) (array []interface{}, ok bool) {
-	switch context := context.(type) {
-	case []interface{}:
-		array = context
-		ok = true
-	default:
-		array = make([]interface{}, 0)
-		ok = false
-	}
-	return
-}
-
-func (n *navigator) writeMap(context interface{}, paths []string, value interface{}) interface{} {
-	n.log.Debugf("writeMap with path %v for %v to set value %v\n", paths, context, value)
-
-	mapSlice := n.getMapSlice(context)
-
-	if len(paths) == 0 {
-		return context
-	}
-
-	children := n.entriesInSlice(mapSlice, paths[0])
-
-	if len(children) == 0 && paths[0] == "*" {
-		n.log.Debugf("\tNo matches, return map as is")
-		return context
-	}
-
-	if len(children) == 0 {
-		newChild := yaml.MapItem{Key: paths[0]}
-		mapSlice = append(mapSlice, newChild)
-		children = n.entriesInSlice(mapSlice, paths[0])
-		n.log.Debugf("\tAppended child at %v for mapSlice %v\n", paths[0], mapSlice)
-	}
-
-	remainingPaths := paths[1:]
-	for _, child := range children {
-		child.Value = n.UpdatedChildValue(child.Value, remainingPaths, value)
-	}
-	n.log.Debugf("\tReturning mapSlice %v\n", mapSlice)
-	return mapSlice
-}
-
-func (n *navigator) writeArray(context interface{}, paths []string, value interface{}) []interface{} {
-	n.log.Debugf("writeArray with path %v for %v to set value %v\n", paths, context, value)
-	array, _ := n.getArray(context)
-
-	if len(paths) == 0 {
-		return array
-	}
-
-	n.log.Debugf("\tarray %v\n", array)
-
-	rawIndex := paths[0]
-	remainingPaths := paths[1:]
-	var index int64
-	// the append array indicator
-	if rawIndex == "+" {
-		index = int64(len(array))
-	} else if rawIndex == "*" {
-		for index, oldChild := range array {
-			array[index] = n.UpdatedChildValue(oldChild, remainingPaths, value)
-		}
-		return array
-	} else {
-		index, _ = strconv.ParseInt(rawIndex, 10, 64) // nolint
-		// writeArray is only called by UpdatedChildValue which handles parsing the
-		// index, as such this renders this dead code.
-	}
-
-	for index >= int64(len(array)) {
-		array = append(array, nil)
-	}
-	currentChild := array[index]
-
-	n.log.Debugf("\tcurrentChild %v\n", currentChild)
-
-	array[index] = n.UpdatedChildValue(currentChild, remainingPaths, value)
-	n.log.Debugf("\tReturning array %v\n", array)
-	return array
-}
-
-func (n *navigator) readMap(context yaml.MapSlice, head string, tail []string) (interface{}, error) {
-	n.log.Debugf("readingMap %v with key %v\n", context, head)
-	if head == "*" {
-		return n.readMapSplat(context, tail)
-	}
-
-	entries := n.entriesInSlice(context, head)
-	if len(entries) == 1 {
-		return n.calculateValue(entries[0].Value, tail)
-	} else if len(entries) == 0 {
-		return nil, nil
-	}
-	var errInIdx error
-	values := make([]interface{}, len(entries))
-	for idx, entry := range entries {
-		values[idx], errInIdx = n.calculateValue(entry.Value, tail)
-		if errInIdx != nil {
-			n.log.Errorf("Error updating index %v in %v", idx, context)
-			return nil, errInIdx
-		}
-
-	}
-	return values, nil
-}
-
-func (n *navigator) readMapSplat(context yaml.MapSlice, tail []string) (interface{}, error) {
-	var newArray = make([]interface{}, len(context))
-	var i = 0
-	for _, entry := range context {
+func (n *navigator) doTraverse(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
+	log.Debug("head %v", head)
+	DebugNode(value)
+	var errorDeepSplatting error
+	if head == "**" && value.Kind != yaml.ScalarNode {
+		errorDeepSplatting = n.recurse(value, head, tail, pathStack)
+		// ignore errors here, we are deep splatting so we may accidently give a string key
+		// to an array sequence
 		if len(tail) > 0 {
-			val, err := n.recurse(entry.Value, tail[0], tail[1:])
-			if err != nil {
-				return nil, err
-			}
-			newArray[i] = val
-		} else {
-			newArray[i] = entry.Value
+			_ = n.recurse(value, tail[0], tail[1:], pathStack)
 		}
-		i++
-	}
-	return newArray, nil
-}
-
-func (n *navigator) readArray(array []interface{}, head int64, tail []string) (interface{}, error) {
-	if head >= int64(len(array)) {
-		return nil, nil
+		return errorDeepSplatting
 	}
 
-	value := array[head]
-	return n.calculateValue(value, tail)
-}
-
-func (n *navigator) readArraySplat(array []interface{}, tail []string) (interface{}, error) {
-	var newArray = make([]interface{}, len(array))
-	for index, value := range array {
-		val, err := n.calculateValue(value, tail)
-		if err != nil {
-			return nil, err
-		}
-		newArray[index] = val
-	}
-	return newArray, nil
-}
-
-func (n *navigator) calculateValue(value interface{}, tail []string) (interface{}, error) {
 	if len(tail) > 0 {
-		return n.recurse(value, tail[0], tail[1:])
+		log.Debugf("diving into %v", tail[0])
+		DebugNode(value)
+		return n.recurse(value, tail[0], tail[1:], pathStack)
 	}
-	return value, nil
+	return n.navigationStrategy.Visit(NewNodeContext(value, head, tail, pathStack))
 }
 
-func (n *navigator) deleteMap(context interface{}, paths []string) (yaml.MapSlice, error) {
-	n.log.Debugf("deleteMap for %v for %v\n", paths, context)
+func (n *navigator) getOrReplace(original *yaml.Node, expectedKind yaml.Kind) *yaml.Node {
+	if original.Kind != expectedKind {
+		log.Debug("wanted %v but it was %v, overriding", expectedKind, original.Kind)
+		return &yaml.Node{Kind: expectedKind}
+	}
+	return original
+}
 
-	mapSlice := n.getMapSlice(context)
+func (n *navigator) recurse(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
+	log.Debug("recursing, processing %v", head)
+	switch value.Kind {
+	case yaml.MappingNode:
+		log.Debug("its a map with %v entries", len(value.Content)/2)
+		return n.recurseMap(value, head, tail, pathStack)
+	case yaml.SequenceNode:
+		log.Debug("its a sequence of %v things!", len(value.Content))
+		if head == "*" || head == "**" {
+			return n.splatArray(value, head, tail, pathStack)
+		} else if head == "+" {
+			return n.appendArray(value, head, tail, pathStack)
+		}
+		return n.recurseArray(value, head, tail, pathStack)
+	case yaml.AliasNode:
+		log.Debug("its an alias!")
+		DebugNode(value.Alias)
+		if n.navigationStrategy.FollowAlias(NewNodeContext(value, head, tail, pathStack)) {
+			log.Debug("following the alias")
+			return n.recurse(value.Alias, head, tail, pathStack)
+		}
+		return nil
+	default:
+		return nil
+	}
+}
 
-	if len(paths) == 0 {
-		return mapSlice, nil
+func (n *navigator) recurseMap(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
+	traversedEntry := false
+	errorVisiting := n.visitMatchingEntries(value, head, tail, pathStack, func(contents []*yaml.Node, indexInMap int) error {
+		log.Debug("recurseMap: visitMatchingEntries")
+		n.navigationStrategy.DebugVisitedNodes()
+		newPathStack := append(pathStack, contents[indexInMap].Value)
+		log.Debug("appended %v", contents[indexInMap].Value)
+		n.navigationStrategy.DebugVisitedNodes()
+		log.Debug("should I traverse? %v, %v", head, pathStackToString(newPathStack))
+		DebugNode(value)
+		if n.navigationStrategy.ShouldTraverse(NewNodeContext(contents[indexInMap+1], head, tail, newPathStack), contents[indexInMap].Value) {
+			log.Debug("recurseMap: Going to traverse")
+			traversedEntry = true
+			// contents[indexInMap+1] = n.getOrReplace(contents[indexInMap+1], guessKind(head, tail, contents[indexInMap+1].Kind))
+			errorTraversing := n.doTraverse(contents[indexInMap+1], head, tail, newPathStack)
+			log.Debug("recurseMap: Finished traversing")
+			n.navigationStrategy.DebugVisitedNodes()
+			return errorTraversing
+		} else {
+			log.Debug("nope not traversing")
+		}
+		return nil
+	})
+
+	if errorVisiting != nil {
+		return errorVisiting
 	}
 
-	var index int
-	var child yaml.MapItem
-	for index, child = range mapSlice {
-		if n.matchesKey(paths[0], child.Key) {
-			n.log.Debugf("\tMatched [%v] with [%v] at index %v", paths[0], child.Key, index)
-			var badDelete error
-			mapSlice, badDelete = n.deleteEntryInMap(mapSlice, child, index, paths)
-			if badDelete != nil {
-				return nil, badDelete
+	if traversedEntry || head == "*" || head == "**" || !n.navigationStrategy.AutoCreateMap(NewNodeContext(value, head, tail, pathStack)) {
+		return nil
+	}
+
+	mapEntryKey := yaml.Node{Value: head, Kind: yaml.ScalarNode}
+	value.Content = append(value.Content, &mapEntryKey)
+	mapEntryValue := yaml.Node{Kind: guessKind(head, tail, 0)}
+	value.Content = append(value.Content, &mapEntryValue)
+	log.Debug("adding new node %v", head)
+	return n.doTraverse(&mapEntryValue, head, tail, append(pathStack, head))
+}
+
+// need to pass the node in, as it may be aliased
+type mapVisitorFn func(contents []*yaml.Node, index int) error
+
+func (n *navigator) visitDirectMatchingEntries(node *yaml.Node, head string, tail []string, pathStack []interface{}, visit mapVisitorFn) error {
+	var contents = node.Content
+	for index := 0; index < len(contents); index = index + 2 {
+		content := contents[index]
+
+		log.Debug("index %v, checking %v, %v", index, content.Value, content.Tag)
+		n.navigationStrategy.DebugVisitedNodes()
+		errorVisiting := visit(contents, index)
+		if errorVisiting != nil {
+			return errorVisiting
+		}
+	}
+	return nil
+}
+
+func (n *navigator) visitMatchingEntries(node *yaml.Node, head string, tail []string, pathStack []interface{}, visit mapVisitorFn) error {
+	var contents = node.Content
+	log.Debug("visitMatchingEntries %v", head)
+	DebugNode(node)
+	// value.Content is a concatenated array of key, value,
+	// so keys are in the even indexes, values in odd.
+	// merge aliases are defined first, but we only want to traverse them
+	// if we don't find a match directly on this node first.
+	errorVisitedDirectEntries := n.visitDirectMatchingEntries(node, head, tail, pathStack, visit)
+
+	if errorVisitedDirectEntries != nil || !n.navigationStrategy.FollowAlias(NewNodeContext(node, head, tail, pathStack)) {
+		return errorVisitedDirectEntries
+	}
+	return n.visitAliases(contents, head, tail, pathStack, visit)
+}
+
+func (n *navigator) visitAliases(contents []*yaml.Node, head string, tail []string, pathStack []interface{}, visit mapVisitorFn) error {
+	// merge aliases are defined first, but we only want to traverse them
+	// if we don't find a match on this node first.
+	// traverse them backwards so that the last alias overrides the preceding.
+	// a node can either be
+	// an alias to one other node (e.g. <<: *blah)
+	// or a sequence of aliases   (e.g. <<: [*blah, *foo])
+	log.Debug("checking for aliases")
+	for index := len(contents) - 2; index >= 0; index = index - 2 {
+
+		if contents[index+1].Kind == yaml.AliasNode {
+			valueNode := contents[index+1]
+			log.Debug("found an alias")
+			DebugNode(contents[index])
+			DebugNode(valueNode)
+
+			errorInAlias := n.visitMatchingEntries(valueNode.Alias, head, tail, pathStack, visit)
+			if errorInAlias != nil {
+				return errorInAlias
+			}
+		} else if contents[index+1].Kind == yaml.SequenceNode {
+			// could be an array of aliases...
+			errorVisitingAliasSeq := n.visitAliasSequence(contents[index+1].Content, head, tail, pathStack, visit)
+			if errorVisitingAliasSeq != nil {
+				return errorVisitingAliasSeq
 			}
 		}
 	}
-
-	return mapSlice, nil
-
+	return nil
 }
 
-func (n *navigator) deleteEntryInMap(original yaml.MapSlice, child yaml.MapItem, index int, paths []string) (yaml.MapSlice, error) {
-	remainingPaths := paths[1:]
-
-	var newSlice yaml.MapSlice
-	if len(remainingPaths) > 0 {
-		newChild := yaml.MapItem{Key: child.Key}
-		var errorDeleting error
-		newChild.Value, errorDeleting = n.DeleteChildValue(child.Value, remainingPaths)
-		if errorDeleting != nil {
-			return nil, errorDeleting
-		}
-
-		newSlice = make(yaml.MapSlice, len(original))
-		for i := range original {
-			item := original[i]
-			if i == index {
-				item = newChild
+func (n *navigator) visitAliasSequence(possibleAliasArray []*yaml.Node, head string, tail []string, pathStack []interface{}, visit mapVisitorFn) error {
+	// need to search this backwards too, so that aliases defined last override the preceding.
+	for aliasIndex := len(possibleAliasArray) - 1; aliasIndex >= 0; aliasIndex = aliasIndex - 1 {
+		child := possibleAliasArray[aliasIndex]
+		if child.Kind == yaml.AliasNode {
+			log.Debug("found an alias")
+			DebugNode(child)
+			errorInAlias := n.visitMatchingEntries(child.Alias, head, tail, pathStack, visit)
+			if errorInAlias != nil {
+				return errorInAlias
 			}
-			newSlice[i] = item
 		}
-	} else {
-		// Delete item from slice at index
-		newSlice = append(original[:index], original[index+1:]...)
-		n.log.Debugf("\tDeleted item index %d from original", index)
 	}
-
-	n.log.Debugf("\tReturning original %v\n", original)
-	return newSlice, nil
+	return nil
 }
 
-func (n *navigator) deleteArraySplat(array []interface{}, tail []string) (interface{}, error) {
-	n.log.Debugf("deleteArraySplat for %v for %v\n", tail, array)
-	var newArray = make([]interface{}, len(array))
-	for index, value := range array {
-		val, err := n.DeleteChildValue(value, tail)
+func (n *navigator) splatArray(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
+	for index, childValue := range value.Content {
+		log.Debug("processing")
+		DebugNode(childValue)
+		childValue = n.getOrReplace(childValue, guessKind(head, tail, childValue.Kind))
+		var err = n.doTraverse(childValue, head, tail, append(pathStack, index))
 		if err != nil {
-			return nil, err
+			return err
 		}
-		newArray[index] = val
 	}
-	return newArray, nil
+	return nil
 }
 
-func (n *navigator) deleteArray(array []interface{}, paths []string, index int64) (interface{}, error) {
-	n.log.Debugf("deleteArray for %v for %v\n", paths, array)
-
-	if index >= int64(len(array)) {
-		return array, nil
-	}
-
-	remainingPaths := paths[1:]
-	if len(remainingPaths) > 0 {
-		// recurse into the array element at index
-		var errorDeleting error
-		array[index], errorDeleting = n.deleteMap(array[index], remainingPaths)
-		if errorDeleting != nil {
-			return nil, errorDeleting
-		}
-
-	} else {
-		// Delete the array element at index
-		array = append(array[:index], array[index+1:]...)
-		n.log.Debugf("\tDeleted item index %d from array, leaving %v", index, array)
-	}
-
-	n.log.Debugf("\tReturning array: %v\n", array)
-	return array, nil
+func (n *navigator) appendArray(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
+	var newNode = yaml.Node{Kind: guessKind(head, tail, 0)}
+	value.Content = append(value.Content, &newNode)
+	log.Debug("appending a new node, %v", value.Content)
+	return n.doTraverse(&newNode, head, tail, append(pathStack, len(value.Content)-1))
+}
+
+func (n *navigator) recurseArray(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
+	var index, err = strconv.ParseInt(head, 10, 64) // nolint
+	if err != nil {
+		return errors.Wrapf(err, "Error parsing array index '%v' for '%v'", head, pathStackToString(pathStack))
+	}
+
+	for int64(len(value.Content)) <= index {
+		value.Content = append(value.Content, &yaml.Node{Kind: guessKind(head, tail, 0)})
+	}
+
+	value.Content[index] = n.getOrReplace(value.Content[index], guessKind(head, tail, value.Content[index].Kind))
+
+	return n.doTraverse(value.Content[index], head, tail, append(pathStack, index))
 }

pkg/yqlib/data_navigator_test.go

@@ -1,397 +1 @@
 package yqlib
-
-import (
-	"fmt"
-	"sort"
-	"testing"
-
-	"github.com/mikefarah/yq/v2/test"
-	logging "gopkg.in/op/go-logging.v1"
-)
-
-func TestDataNavigator(t *testing.T) {
-	var log = logging.MustGetLogger("yq")
-	subject := NewDataNavigator(log)
-
-	t.Run("TestReadMap_simple", func(t *testing.T) {
-		var data = test.ParseData(`
----
-b:
-  c: 2
-`)
-		got, _ := subject.ReadChildValue(data, []string{"b", "c"})
-		test.AssertResult(t, 2, got)
-	})
-
-	t.Run("TestReadMap_numberKey", func(t *testing.T) {
-		var data = test.ParseData(`
----
-200: things
-`)
-		got, _ := subject.ReadChildValue(data, []string{"200"})
-		test.AssertResult(t, "things", got)
-	})
-
-	t.Run("TestReadMap_splat", func(t *testing.T) {
-		var data = test.ParseData(`
----
-mapSplat:
-  item1: things
-  item2: whatever
-  otherThing: cat
-`)
-		res, _ := subject.ReadChildValue(data, []string{"mapSplat", "*"})
-		test.AssertResult(t, "[things whatever cat]", fmt.Sprintf("%v", res))
-	})
-
-	t.Run("TestReadMap_prefixSplat", func(t *testing.T) {
-		var data = test.ParseData(`
----
-mapSplat:
-  item1: things
-  item2: whatever
-  otherThing: cat
-`)
-		res, _ := subject.ReadChildValue(data, []string{"mapSplat", "item*"})
-		test.AssertResult(t, "[things whatever]", fmt.Sprintf("%v", res))
-	})
-
-	t.Run("TestReadMap_deep_splat", func(t *testing.T) {
-		var data = test.ParseData(`
----
-mapSplatDeep:
-  item1:
-    cats: bananas
-  item2:
-    cats: apples
-`)
-
-		res, _ := subject.ReadChildValue(data, []string{"mapSplatDeep", "*", "cats"})
-		result := res.([]interface{})
-		var actual = []string{result[0].(string), result[1].(string)}
-		sort.Strings(actual)
-		test.AssertResult(t, "[apples bananas]", fmt.Sprintf("%v", actual))
-	})
-
-	t.Run("TestReadMap_key_doesnt_exist", func(t *testing.T) {
-		var data = test.ParseData(`
----
-b:
-  c: 2
-`)
-		got, _ := subject.ReadChildValue(data, []string{"b", "x", "f", "c"})
-		test.AssertResult(t, nil, got)
-	})
-
-	t.Run("TestReadMap_recurse_against_string", func(t *testing.T) {
-		var data = test.ParseData(`
----
-a: cat
-`)
-		got, _ := subject.ReadChildValue(data, []string{"a", "b"})
-		test.AssertResult(t, nil, got)
-	})
-
-	t.Run("TestReadMap_with_array", func(t *testing.T) {
-		var data = test.ParseData(`
----
-b:
-  d:
-    - 3
-    - 4
-`)
-		got, _ := subject.ReadChildValue(data, []string{"b", "d", "1"})
-		test.AssertResult(t, 4, got)
-	})
-
-	t.Run("TestReadMap_with_array_and_bad_index", func(t *testing.T) {
-		var data = test.ParseData(`
----
-b:
-  d:
-    - 3
-    - 4
-`)
-		_, err := subject.ReadChildValue(data, []string{"b", "d", "x"})
-		if err == nil {
-			t.Fatal("Expected error due to invalid path")
-		}
-		expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
-		test.AssertResult(t, expectedOutput, err.Error())
-	})
-
-	t.Run("TestReadMap_with_mapsplat_array_and_bad_index", func(t *testing.T) {
-		var data = test.ParseData(`
----
-b:
-  d:
-    e:
-      - 3
-      - 4
-    f:
-      - 1
-      - 2
-`)
-		_, err := subject.ReadChildValue(data, []string{"b", "d", "*", "x"})
-		if err == nil {
-			t.Fatal("Expected error due to invalid path")
-		}
-		expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
-		test.AssertResult(t, expectedOutput, err.Error())
-	})
-
-	t.Run("TestReadMap_with_arraysplat_map_array_and_bad_index", func(t *testing.T) {
-		var data = test.ParseData(`
----
-b:
-  d:
-    - names:
-        - fred
-        - smith
-    - names:
-        - sam
-        - bo
-`)
-		_, err := subject.ReadChildValue(data, []string{"b", "d", "*", "names", "x"})
-		if err == nil {
-			t.Fatal("Expected error due to invalid path")
-		}
-		expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
-		test.AssertResult(t, expectedOutput, err.Error())
-	})
-
-	t.Run("TestReadMap_with_array_out_of_bounds", func(t *testing.T) {
-		var data = test.ParseData(`
----
-b:
-  d:
-    - 3
-    - 4
-`)
-		got, _ := subject.ReadChildValue(data, []string{"b", "d", "3"})
-		test.AssertResult(t, nil, got)
-	})
-
-	t.Run("TestReadMap_with_array_out_of_bounds_by_1", func(t *testing.T) {
-		var data = test.ParseData(`
----
-b:
-  d:
-    - 3
-    - 4
-`)
-		got, _ := subject.ReadChildValue(data, []string{"b", "d", "2"})
-		test.AssertResult(t, nil, got)
-	})
-
-	t.Run("TestReadMap_with_array_splat", func(t *testing.T) {
-		var data = test.ParseData(`
-e:
-  -
-    name: Fred
-    thing: cat
-  -
-    name: Sam
-    thing: dog
-`)
-		got, _ := subject.ReadChildValue(data, []string{"e", "*", "name"})
-		test.AssertResult(t, "[Fred Sam]", fmt.Sprintf("%v", got))
-	})
-
-	t.Run("TestWrite_really_simple", func(t *testing.T) {
-		var data = test.ParseData(`
-b: 2
-`)
-
-		updated := subject.UpdatedChildValue(data, []string{"b"}, "4")
-		test.AssertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated))
-	})
-
-	t.Run("TestWrite_simple", func(t *testing.T) {
-		var data = test.ParseData(`
-b:
-  c: 2
-`)
-
-		updated := subject.UpdatedChildValue(data, []string{"b", "c"}, "4")
-		test.AssertResult(t, "[{b [{c 4}]}]", fmt.Sprintf("%v", updated))
-	})
-
-	t.Run("TestWrite_new", func(t *testing.T) {
-		var data = test.ParseData(`
-b:
-  c: 2
-`)
-
-		updated := subject.UpdatedChildValue(data, []string{"b", "d"}, "4")
-		test.AssertResult(t, "[{b [{c 2} {d 4}]}]", fmt.Sprintf("%v", updated))
-	})
-
-	t.Run("TestWrite_new_deep", func(t *testing.T) {
-		var data = test.ParseData(`
-b:
-  c: 2
-`)
-
-		updated := subject.UpdatedChildValue(data, []string{"b", "d", "f"}, "4")
-		test.AssertResult(t, "[{b [{c 2} {d [{f 4}]}]}]", fmt.Sprintf("%v", updated))
-	})
-
-	t.Run("TestWrite_array", func(t *testing.T) {
-		var data = test.ParseData(`
-b:
-  - aa
-`)
-
-		updated := subject.UpdatedChildValue(data, []string{"b", "0"}, "bb")
-
-		test.AssertResult(t, "[{b [bb]}]", fmt.Sprintf("%v", updated))
-	})
-
-	t.Run("TestWrite_new_array", func(t *testing.T) {
-		var data = test.ParseData(`
-b:
-  c: 2
-`)
-
-		updated := subject.UpdatedChildValue(data, []string{"b", "0"}, "4")
-		test.AssertResult(t, "[{b [{c 2} {0 4}]}]", fmt.Sprintf("%v", updated))
-	})
-
-	t.Run("TestWrite_new_array_deep", func(t *testing.T) {
-		var data = test.ParseData(`
-a: apple
-`)
-
-		updated := subject.UpdatedChildValue(data, []string{"b", "+", "c"}, "4")
-		test.AssertResult(t, "[{a apple} {b [[{c 4}]]}]", fmt.Sprintf("%v", updated))
-	})
-
-	t.Run("TestWrite_new_map_array_deep", func(t *testing.T) {
-		var data = test.ParseData(`
-b:
-  c: 2
-`)
-
-		updated := subject.UpdatedChildValue(data, []string{"b", "d", "+"}, "4")
-		test.AssertResult(t, "[{b [{c 2} {d [4]}]}]", fmt.Sprintf("%v", updated))
-	})
-
-	t.Run("TestWrite_add_to_array", func(t *testing.T) {
-		var data = test.ParseData(`
-b:
-  - aa
-`)
-
-		updated := subject.UpdatedChildValue(data, []string{"b", "1"}, "bb")
-		test.AssertResult(t, "[{b [aa bb]}]", fmt.Sprintf("%v", updated))
-	})
-
-	t.Run("TestWrite_with_no_tail", func(t *testing.T) {
-		var data = test.ParseData(`
-b:
-  c: 2
-`)
-		updated := subject.UpdatedChildValue(data, []string{"b"}, "4")
-
-		test.AssertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated))
-	})
-
-	t.Run("TestWriteMap_no_paths", func(t *testing.T) {
-		var data = test.ParseData(`
-b: 5
-`)
-		var new = test.ParseData(`
-c: 4
-`)
-		result := subject.UpdatedChildValue(data, []string{}, new)
-		test.AssertResult(t, fmt.Sprintf("%v", new), fmt.Sprintf("%v", result))
-	})
-
-	t.Run("TestWriteArray_no_paths", func(t *testing.T) {
-		var data = make([]interface{}, 1)
-		data[0] = "mike"
-		var new = test.ParseData(`
-c: 4
-`)
-		result := subject.UpdatedChildValue(data, []string{}, new)
-		test.AssertResult(t, fmt.Sprintf("%v", new), fmt.Sprintf("%v", result))
-	})
-
-	t.Run("TestDelete_MapItem", func(t *testing.T) {
-		var data = test.ParseData(`
-a: 123
-b: 456
-`)
-		var expected = test.ParseData(`
-b: 456
-`)
-
-		result, _ := subject.DeleteChildValue(data, []string{"a"})
-		test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result))
-	})
-
-	// Ensure deleting an index into a string does nothing
-	t.Run("TestDelete_index_to_string", func(t *testing.T) {
-		var data = test.ParseData(`
-a: mystring
-`)
-		result, _ := subject.DeleteChildValue(data, []string{"a", "0"})
-		test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
-	})
-
-	t.Run("TestDelete_list_index", func(t *testing.T) {
-		var data = test.ParseData(`
-a: [3, 4]
-`)
-		var expected = test.ParseData(`
-a: [3]
-`)
-		result, _ := subject.DeleteChildValue(data, []string{"a", "1"})
-		test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result))
-	})
-
-	t.Run("TestDelete_list_index_beyond_bounds", func(t *testing.T) {
-		var data = test.ParseData(`
-a: [3, 4]
-`)
-		result, _ := subject.DeleteChildValue(data, []string{"a", "5"})
-		test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
-	})
-
-	t.Run("TestDelete_list_index_out_of_bounds_by_1", func(t *testing.T) {
-		var data = test.ParseData(`
-a: [3, 4]
-`)
-		result, _ := subject.DeleteChildValue(data, []string{"a", "2"})
-		test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
-	})
-
-	t.Run("TestDelete_no_paths", func(t *testing.T) {
-		var data = test.ParseData(`
-a: [3, 4]
-b:
-  - name: test
-`)
-		result, _ := subject.DeleteChildValue(data, []string{})
-		test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
-	})
-
-	t.Run("TestDelete_array_map_item", func(t *testing.T) {
-		var data = test.ParseData(`
-b:
-- name: fred
-  value: blah
-- name: john
-  value: test
-`)
-		var expected = test.ParseData(`
-b:
-- value: blah
-- name: john
-  value: test
-`)
-		result, _ := subject.DeleteChildValue(data, []string{"b", "0", "name"})
-		test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result))
-	})
-}

pkg/yqlib/delete_navigation_strategy.go

@@ -0,0 +1,66 @@
+package yqlib
+
+import (
+	"strconv"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func DeleteNavigationStrategy(pathElementToDelete string) NavigationStrategy {
+	parser := NewPathParser()
+	return &NavigationStrategyImpl{
+		visitedNodes: []*NodeContext{},
+		followAlias: func(nodeContext NodeContext) bool {
+			return false
+		},
+		autoCreateMap: func(nodeContext NodeContext) bool {
+			return true
+		},
+		visit: func(nodeContext NodeContext) error {
+			node := nodeContext.Node
+			log.Debug("need to find and delete %v in here", pathElementToDelete)
+			DebugNode(node)
+			if node.Kind == yaml.SequenceNode {
+				newContent, errorDeleting := deleteFromArray(node.Content, pathElementToDelete)
+				if errorDeleting != nil {
+					return errorDeleting
+				}
+				node.Content = newContent
+			} else if node.Kind == yaml.MappingNode {
+				node.Content = deleteFromMap(parser, node.Content, nodeContext.PathStack, pathElementToDelete)
+			}
+			return nil
+		},
+	}
+}
+func deleteFromMap(pathParser PathParser, contents []*yaml.Node, pathStack []interface{}, pathElementToDelete string) []*yaml.Node {
+	newContents := make([]*yaml.Node, 0)
+	for index := 0; index < len(contents); index = index + 2 {
+		keyNode := contents[index]
+		valueNode := contents[index+1]
+		if !pathParser.MatchesNextPathElement(NewNodeContext(keyNode, pathElementToDelete, []string{}, pathStack), keyNode.Value) {
+			log.Debug("adding node %v", keyNode.Value)
+			newContents = append(newContents, keyNode, valueNode)
+		} else {
+			log.Debug("skipping node %v", keyNode.Value)
+		}
+	}
+	return newContents
+}
+
+func deleteFromArray(content []*yaml.Node, pathElementToDelete string) ([]*yaml.Node, error) {
+
+	if pathElementToDelete == "*" {
+		return make([]*yaml.Node, 0), nil
+	}
+
+	var index, err = strconv.ParseInt(pathElementToDelete, 10, 64) // nolint
+	if err != nil {
+		return content, err
+	}
+	if index >= int64(len(content)) {
+		log.Debug("index %v is greater than content length %v", index, len(content))
+		return content, nil
+	}
+	return append(content[:index], content[index+1:]...), nil
+}

pkg/yqlib/encoder.go

@@ -0,0 +1,44 @@
+package yqlib
+
+import (
+	"encoding/json"
+	"io"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type Encoder interface {
+	Encode(node *yaml.Node) error
+}
+
+type yamlEncoder struct {
+	encoder *yaml.Encoder
+}
+
+func NewYamlEncoder(destination io.Writer) Encoder {
+	var encoder = yaml.NewEncoder(destination)
+	encoder.SetIndent(2)
+	return &yamlEncoder{encoder}
+}
+
+func (ye *yamlEncoder) Encode(node *yaml.Node) error {
+	return ye.encoder.Encode(node)
+}
+
+type jsonEncoder struct {
+	encoder *json.Encoder
+}
+
+func NewJsonEncoder(destination io.Writer) Encoder {
+	var encoder = json.NewEncoder(destination)
+	return &jsonEncoder{encoder}
+}
+
+func (je *jsonEncoder) Encode(node *yaml.Node) error {
+	var dataBucket interface{}
+	errorDecoding := node.Decode(&dataBucket)
+	if errorDecoding != nil {
+		return errorDecoding
+	}
+	return je.encoder.Encode(dataBucket)
+}

pkg/yqlib/lib.go

@@ -1,70 +1,148 @@
 package yqlib
 
 import (
-	mergo "gopkg.in/imdario/mergo.v0"
+	"bytes"
+	"fmt"
+	"strconv"
+	"strings"
+
 	logging "gopkg.in/op/go-logging.v1"
+	yaml "gopkg.in/yaml.v3"
 )
 
+var log = logging.MustGetLogger("yq")
+
+type UpdateCommand struct {
+	Command   string
+	Path      string
+	Value     *yaml.Node
+	Overwrite bool
+}
+
+func DebugNode(value *yaml.Node) {
+	if value == nil {
+		log.Debug("-- node is nil --")
+	} else if log.IsEnabledFor(logging.DEBUG) {
+		buf := new(bytes.Buffer)
+		encoder := yaml.NewEncoder(buf)
+		errorEncoding := encoder.Encode(value)
+		if errorEncoding != nil {
+			log.Error("Error debugging node, %v", errorEncoding.Error())
+		}
+		encoder.Close()
+		log.Debug("Tag: %v", value.Tag)
+		log.Debug("%v", buf.String())
+	}
+}
+
+func pathStackToString(pathStack []interface{}) string {
+	return mergePathStackToString(pathStack, false)
+}
+
+func mergePathStackToString(pathStack []interface{}, appendArrays bool) string {
+	var sb strings.Builder
+	for index, path := range pathStack {
+		switch path.(type) {
+		case int:
+			if appendArrays {
+				sb.WriteString("[+]")
+			} else {
+				sb.WriteString(fmt.Sprintf("[%v]", path))
+			}
+
+		default:
+			sb.WriteString(fmt.Sprintf("%v", path))
+		}
+
+		if index < len(pathStack)-1 {
+			sb.WriteString(".")
+		}
+	}
+	return sb.String()
+}
+
+func guessKind(head string, tail []string, guess yaml.Kind) yaml.Kind {
+	log.Debug("tail %v", tail)
+	if len(tail) == 0 && guess == 0 {
+		log.Debug("end of path, must be a scalar")
+		return yaml.ScalarNode
+	} else if len(tail) == 0 {
+		return guess
+	}
+
+	var _, errorParsingInt = strconv.ParseInt(tail[0], 10, 64)
+	if tail[0] == "+" || errorParsingInt == nil {
+		return yaml.SequenceNode
+	}
+	if (tail[0] == "*" || tail[0] == "**" || head == "**") && (guess == yaml.SequenceNode || guess == yaml.MappingNode) {
+		return guess
+	}
+	if guess == yaml.AliasNode {
+		log.Debug("guess was an alias, okey doke.")
+		return guess
+	}
+	log.Debug("forcing a mapping node")
+	log.Debug("yaml.SequenceNode %v", guess == yaml.SequenceNode)
+	log.Debug("yaml.ScalarNode %v", guess == yaml.ScalarNode)
+	return yaml.MappingNode
+}
+
 type YqLib interface {
-	ReadPath(dataBucket interface{}, path string) (interface{}, error)
-	WritePath(dataBucket interface{}, path string, value interface{}) interface{}
-	PrefixPath(dataBucket interface{}, prefix string) interface{}
-	DeletePath(dataBucket interface{}, path string) (interface{}, error)
-	Merge(dst interface{}, src interface{}, overwrite bool, append bool) error
+	Get(rootNode *yaml.Node, path string) ([]*NodeContext, error)
+	Update(rootNode *yaml.Node, updateCommand UpdateCommand, autoCreate bool) error
+	New(path string) yaml.Node
+
+	PathStackToString(pathStack []interface{}) string
+	MergePathStackToString(pathStack []interface{}, appendArrays bool) string
 }
 
 type lib struct {
-	navigator DataNavigator
-	parser    PathParser
+	parser PathParser
 }
 
-func NewYqLib(l *logging.Logger) YqLib {
+func NewYqLib() YqLib {
 	return &lib{
-		navigator: NewDataNavigator(l),
-		parser:    NewPathParser(),
+		parser: NewPathParser(),
 	}
 }
 
-func (l *lib) ReadPath(dataBucket interface{}, path string) (interface{}, error) {
+func (l *lib) Get(rootNode *yaml.Node, path string) ([]*NodeContext, error) {
 	var paths = l.parser.ParsePath(path)
-	return l.navigator.ReadChildValue(dataBucket, paths)
+	NavigationStrategy := ReadNavigationStrategy()
+	navigator := NewDataNavigator(NavigationStrategy)
+	error := navigator.Traverse(rootNode, paths)
+	return NavigationStrategy.GetVisitedNodes(), error
+
 }
 
-func (l *lib) WritePath(dataBucket interface{}, path string, value interface{}) interface{} {
+func (l *lib) PathStackToString(pathStack []interface{}) string {
+	return pathStackToString(pathStack)
+}
+
+func (l *lib) MergePathStackToString(pathStack []interface{}, appendArrays bool) string {
+	return mergePathStackToString(pathStack, appendArrays)
+}
+
+func (l *lib) New(path string) yaml.Node {
 	var paths = l.parser.ParsePath(path)
-	return l.navigator.UpdatedChildValue(dataBucket, paths, value)
+	newNode := yaml.Node{Kind: guessKind("", paths, 0)}
+	return newNode
 }
 
-func (l *lib) PrefixPath(dataBucket interface{}, prefix string) interface{} {
-	var paths = l.parser.ParsePath(prefix)
-
-	// Inverse order
-	for i := len(paths)/2 - 1; i >= 0; i-- {
-		opp := len(paths) - 1 - i
-		paths[i], paths[opp] = paths[opp], paths[i]
+func (l *lib) Update(rootNode *yaml.Node, updateCommand UpdateCommand, autoCreate bool) error {
+	log.Debugf("%v to %v", updateCommand.Command, updateCommand.Path)
+	switch updateCommand.Command {
+	case "update":
+		var paths = l.parser.ParsePath(updateCommand.Path)
+		navigator := NewDataNavigator(UpdateNavigationStrategy(updateCommand, autoCreate))
+		return navigator.Traverse(rootNode, paths)
+	case "delete":
+		var paths = l.parser.ParsePath(updateCommand.Path)
+		lastBit, newTail := paths[len(paths)-1], paths[:len(paths)-1]
+		navigator := NewDataNavigator(DeleteNavigationStrategy(lastBit))
+		return navigator.Traverse(rootNode, newTail)
+	default:
+		return fmt.Errorf("Unknown command %v", updateCommand.Command)
 	}
 
-	var mapDataBucket = dataBucket
-	for _, key := range paths {
-		singlePath := []string{key}
-		mapDataBucket = l.navigator.UpdatedChildValue(nil, singlePath, mapDataBucket)
-	}
-
-	return mapDataBucket
-}
-
-func (l *lib) DeletePath(dataBucket interface{}, path string) (interface{}, error) {
-	var paths = l.parser.ParsePath(path)
-	return l.navigator.DeleteChildValue(dataBucket, paths)
-}
-
-func (l *lib) Merge(dst interface{}, src interface{}, overwriteFlag bool, appendFlag bool) error {
-	opts := []func(*mergo.Config){}
-	if overwriteFlag {
-		opts = append(opts, mergo.WithOverride)
-	}
-	if appendFlag {
-		opts = append(opts, mergo.WithAppendSlice)
-	}
-	return mergo.Merge(dst, src, opts...)
 }

pkg/yqlib/lib_test.go

@@ -1,183 +1,176 @@
 package yqlib
 
 import (
-	"fmt"
 	"testing"
 
-	"github.com/mikefarah/yq/v2/test"
-	logging "gopkg.in/op/go-logging.v1"
+	"github.com/mikefarah/yq/v3/test"
 )
 
 func TestLib(t *testing.T) {
 
-	var log = logging.MustGetLogger("yq")
-	subject := NewYqLib(log)
+	subject := NewYqLib()
 
-	t.Run("TestReadPath", func(t *testing.T) {
-		var data = test.ParseData(`
----
-b:
-  2: c
-`)
-
-		got, _ := subject.ReadPath(data, "b.2")
-		test.AssertResult(t, `c`, got)
+	t.Run("PathStackToString_Empty", func(t *testing.T) {
+		emptyArray := make([]interface{}, 0)
+		got := subject.PathStackToString(emptyArray)
+		test.AssertResult(t, ``, got)
 	})
 
-	t.Run("TestReadPath_WithError", func(t *testing.T) {
-		var data = test.ParseData(`
----
-b:
-  - c
-`)
-
-		_, err := subject.ReadPath(data, "b.[a]")
-		if err == nil {
-			t.Fatal("Expected error due to invalid path")
-		}
+	t.Run("PathStackToString", func(t *testing.T) {
+		array := make([]interface{}, 3)
+		array[0] = "a"
+		array[1] = 0
+		array[2] = "b"
+		got := subject.PathStackToString(array)
+		test.AssertResult(t, `a.[0].b`, got)
 	})
 
-	t.Run("TestWritePath", func(t *testing.T) {
-		var data = test.ParseData(`
----
-b:
-  2: c
-`)
-
-		got := subject.WritePath(data, "b.3", "a")
-		test.AssertResult(t, `[{b [{2 c} {3 a}]}]`, fmt.Sprintf("%v", got))
+	t.Run("MergePathStackToString", func(t *testing.T) {
+		array := make([]interface{}, 3)
+		array[0] = "a"
+		array[1] = 0
+		array[2] = "b"
+		got := subject.MergePathStackToString(array, true)
+		test.AssertResult(t, `a.[+].b`, got)
 	})
 
-	t.Run("TestPrefixPath", func(t *testing.T) {
-		var data = test.ParseData(`
----
-b:
-  2: c
-`)
+	// 	t.Run("TestReadPath_WithError", func(t *testing.T) {
+	// 		var data = test.ParseData(`
+	// ---
+	// b:
+	//   - c
+	// `)
 
-		got := subject.PrefixPath(data, "a.d")
-		test.AssertResult(t, `[{a [{d [{b [{2 c}]}]}]}]`, fmt.Sprintf("%v", got))
-	})
+	// 		_, err := subject.ReadPath(data, "b.[a]")
+	// 		if err == nil {
+	// 			t.Fatal("Expected error due to invalid path")
+	// 		}
+	// 	})
 
-	t.Run("TestDeletePath", func(t *testing.T) {
-		var data = test.ParseData(`
----
-b:
-  2: c
-  3: a
-`)
+	// 	t.Run("TestWritePath", func(t *testing.T) {
+	// 		var data = test.ParseData(`
+	// ---
+	// b:
+	//   2: c
+	// `)
 
-		got, _ := subject.DeletePath(data, "b.2")
-		test.AssertResult(t, `[{b [{3 a}]}]`, fmt.Sprintf("%v", got))
-	})
+	// 		got := subject.WritePath(data, "b.3", "a")
+	// 		test.AssertResult(t, `[{b [{2 c} {3 a}]}]`, fmt.Sprintf("%v", got))
+	// 	})
 
-	t.Run("TestDeletePath_WithError", func(t *testing.T) {
-		var data = test.ParseData(`
----
-b:
-  - c
-`)
+	// 	t.Run("TestPrefixPath", func(t *testing.T) {
+	// 		var data = test.ParseData(`
+	// ---
+	// b:
+	//   2: c
+	// `)
 
-		_, err := subject.DeletePath(data, "b.[a]")
-		if err == nil {
-			t.Fatal("Expected error due to invalid path")
-		}
-	})
+	// 		got := subject.PrefixPath(data, "a.d")
+	// 		test.AssertResult(t, `[{a [{d [{b [{2 c}]}]}]}]`, fmt.Sprintf("%v", got))
+	// 	})
 
-	t.Run("TestMerge", func(t *testing.T) {
-		var dst = test.ParseData(`
----
-a: b
-c: d
-`)
-		var src = test.ParseData(`
----
-a: 1
-b: 2
-`)
+	// 	t.Run("TestDeletePath", func(t *testing.T) {
+	// 		var data = test.ParseData(`
+	// ---
+	// b:
+	//   2: c
+	//   3: a
+	// `)
 
-		var mergedData = make(map[interface{}]interface{})
-		mergedData["root"] = dst
-		var mapDataBucket = make(map[interface{}]interface{})
-		mapDataBucket["root"] = src
+	// 		got, _ := subject.DeletePath(data, "b.2")
+	// 		test.AssertResult(t, `[{b [{3 a}]}]`, fmt.Sprintf("%v", got))
+	// 	})
 
-		err := subject.Merge(&mergedData, mapDataBucket, false, false)
-		if err != nil {
-			t.Fatal("Unexpected error")
-		}
-		test.AssertResult(t, `[{a b} {c d}]`, fmt.Sprintf("%v", mergedData["root"]))
-	})
+	// 	t.Run("TestDeletePath_WithError", func(t *testing.T) {
+	// 		var data = test.ParseData(`
+	// ---
+	// b:
+	//   - c
+	// `)
 
-	t.Run("TestMerge_WithOverwrite", func(t *testing.T) {
-		var dst = test.ParseData(`
----
-a: b
-c: d
-`)
-		var src = test.ParseData(`
----
-a: 1
-b: 2
-`)
+	// 		_, err := subject.DeletePath(data, "b.[a]")
+	// 		if err == nil {
+	// 			t.Fatal("Expected error due to invalid path")
+	// 		}
+	// 	})
 
-		var mergedData = make(map[interface{}]interface{})
-		mergedData["root"] = dst
-		var mapDataBucket = make(map[interface{}]interface{})
-		mapDataBucket["root"] = src
+	// 	t.Run("TestMerge", func(t *testing.T) {
+	// 		var dst = test.ParseData(`
+	// ---
+	// a: b
+	// c: d
+	// `)
+	// 		var src = test.ParseData(`
+	// ---
+	// a: 1
+	// b: 2
+	// `)
 
-		err := subject.Merge(&mergedData, mapDataBucket, true, false)
-		if err != nil {
-			t.Fatal("Unexpected error")
-		}
-		test.AssertResult(t, `[{a 1} {b 2}]`, fmt.Sprintf("%v", mergedData["root"]))
-	})
+	// 		var mergedData = make(map[interface{}]interface{})
+	// 		mergedData["root"] = dst
+	// 		var mapDataBucket = make(map[interface{}]interface{})
+	// 		mapDataBucket["root"] = src
 
-	t.Run("TestMerge_WithAppend", func(t *testing.T) {
-		var dst = test.ParseData(`
----
-a: b
-c: d
-`)
-		var src = test.ParseData(`
----
-a: 1
-b: 2
-`)
+	// 		err := subject.Merge(&mergedData, mapDataBucket, false, false)
+	// 		if err != nil {
+	// 			t.Fatal("Unexpected error")
+	// 		}
+	// 		test.AssertResult(t, `[{a b} {c d}]`, fmt.Sprintf("%v", mergedData["root"]))
+	// 	})
 
-		var mergedData = make(map[interface{}]interface{})
-		mergedData["root"] = dst
-		var mapDataBucket = make(map[interface{}]interface{})
-		mapDataBucket["root"] = src
+	// 	t.Run("TestMerge_WithOverwrite", func(t *testing.T) {
+	// 		var dst = test.ParseData(`
+	// ---
+	// a: b
+	// c: d
+	// `)
+	// 		var src = test.ParseData(`
+	// ---
+	// a: 1
+	// b: 2
+	// `)
 
-		err := subject.Merge(&mergedData, mapDataBucket, false, true)
-		if err != nil {
-			t.Fatal("Unexpected error")
-		}
-		test.AssertResult(t, `[{a b} {c d} {a 1} {b 2}]`, fmt.Sprintf("%v", mergedData["root"]))
-	})
+	// 		var mergedData = make(map[interface{}]interface{})
+	// 		mergedData["root"] = dst
+	// 		var mapDataBucket = make(map[interface{}]interface{})
+	// 		mapDataBucket["root"] = src
 
-	t.Run("TestMerge_WithAppendAndOverwrite", func(t *testing.T) {
-		var dst = map[interface{}]interface{}{
-			"a": "initial",
-			"b": []string{"old"},
-		}
-		var src = map[interface{}]interface{}{
-			"a": "replaced",
-			"b": []string{"new"},
-		}
+	// 		err := subject.Merge(&mergedData, mapDataBucket, true, false)
+	// 		if err != nil {
+	// 			t.Fatal("Unexpected error")
+	// 		}
+	// 		test.AssertResult(t, `[{a 1} {b 2}]`, fmt.Sprintf("%v", mergedData["root"]))
+	// 	})
 
-		err := subject.Merge(&dst, src, true, true)
-		if err != nil {
-			t.Fatal("Unexpected error")
-		}
-		test.AssertResult(t, `map[a:replaced b:[old new]]`, fmt.Sprintf("%v", dst))
-	})
+	// 	t.Run("TestMerge_WithAppend", func(t *testing.T) {
+	// 		var dst = test.ParseData(`
+	// ---
+	// a: b
+	// c: d
+	// `)
+	// 		var src = test.ParseData(`
+	// ---
+	// a: 1
+	// b: 2
+	// `)
 
-	t.Run("TestMerge_WithError", func(t *testing.T) {
-		err := subject.Merge(nil, nil, false, false)
-		if err == nil {
-			t.Fatal("Expected error due to nil")
-		}
-	})
+	// 		var mergedData = make(map[interface{}]interface{})
+	// 		mergedData["root"] = dst
+	// 		var mapDataBucket = make(map[interface{}]interface{})
+	// 		mapDataBucket["root"] = src
+
+	// 		err := subject.Merge(&mergedData, mapDataBucket, false, true)
+	// 		if err != nil {
+	// 			t.Fatal("Unexpected error")
+	// 		}
+	// 		test.AssertResult(t, `[{a b} {c d} {a 1} {b 2}]`, fmt.Sprintf("%v", mergedData["root"]))
+	// 	})
+
+	// 	t.Run("TestMerge_WithError", func(t *testing.T) {
+	// 		err := subject.Merge(nil, nil, false, false)
+	// 		if err == nil {
+	// 			t.Fatal("Expected error due to nil")
+	// 		}
+	// 	})
 
 }

pkg/yqlib/navigation_strategy.go

@@ -0,0 +1,148 @@
+package yqlib
+
+import (
+	"fmt"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type NodeContext struct {
+	Node      *yaml.Node
+	Head      string
+	Tail      []string
+	PathStack []interface{}
+}
+
+func NewNodeContext(node *yaml.Node, head string, tail []string, pathStack []interface{}) NodeContext {
+	newTail := make([]string, len(tail))
+	copy(newTail, tail)
+
+	newPathStack := make([]interface{}, len(pathStack))
+	copy(newPathStack, pathStack)
+	return NodeContext{
+		Node:      node,
+		Head:      head,
+		Tail:      newTail,
+		PathStack: newPathStack,
+	}
+}
+
+type NavigationStrategy interface {
+	FollowAlias(nodeContext NodeContext) bool
+	AutoCreateMap(nodeContext NodeContext) bool
+	Visit(nodeContext NodeContext) error
+	// node key is the string value of the last element in the path stack
+	// we use it to match against the pathExpression in head.
+	ShouldTraverse(nodeContext NodeContext, nodeKey string) bool
+	GetVisitedNodes() []*NodeContext
+	DebugVisitedNodes()
+}
+
+type NavigationStrategyImpl struct {
+	followAlias   func(nodeContext NodeContext) bool
+	autoCreateMap func(nodeContext NodeContext) bool
+	visit         func(nodeContext NodeContext) error
+	visitedNodes  []*NodeContext
+}
+
+func (ns *NavigationStrategyImpl) GetVisitedNodes() []*NodeContext {
+	return ns.visitedNodes
+}
+
+func (ns *NavigationStrategyImpl) FollowAlias(nodeContext NodeContext) bool {
+	return ns.followAlias(nodeContext)
+}
+
+func (ns *NavigationStrategyImpl) AutoCreateMap(nodeContext NodeContext) bool {
+	return ns.autoCreateMap(nodeContext)
+}
+
+func (ns *NavigationStrategyImpl) ShouldTraverse(nodeContext NodeContext, nodeKey string) bool {
+	// we should traverse aliases (if enabled), but not visit them :/
+	if len(nodeContext.PathStack) == 0 {
+		return true
+	}
+
+	if ns.alreadyVisited(nodeContext.PathStack) {
+		return false
+	}
+
+	parser := NewPathParser()
+
+	return (nodeKey == "<<" && ns.FollowAlias(nodeContext)) || (nodeKey != "<<" &&
+		parser.MatchesNextPathElement(nodeContext, nodeKey))
+}
+
+func (ns *NavigationStrategyImpl) shouldVisit(nodeContext NodeContext) bool {
+	pathStack := nodeContext.PathStack
+	if len(pathStack) == 0 {
+		return true
+	}
+	log.Debug("tail len %v", len(nodeContext.Tail))
+	// SOMETHING HERE!
+
+	if ns.alreadyVisited(pathStack) || len(nodeContext.Tail) != 0 {
+		return false
+	}
+
+	nodeKey := fmt.Sprintf("%v", pathStack[len(pathStack)-1])
+	log.Debug("nodeKey: %v, nodeContext.Head: %v", nodeKey, nodeContext.Head)
+	parser := NewPathParser()
+
+	// only visit aliases if its an exact match
+	return (nodeKey == "<<" && nodeContext.Head == "<<") || (nodeKey != "<<" &&
+		parser.MatchesNextPathElement(nodeContext, nodeKey))
+}
+
+func (ns *NavigationStrategyImpl) Visit(nodeContext NodeContext) error {
+	log.Debug("Visit?, %v, %v", nodeContext.Head, pathStackToString(nodeContext.PathStack))
+	DebugNode(nodeContext.Node)
+	if ns.shouldVisit(nodeContext) {
+		log.Debug("yep, visiting")
+		//  pathStack array must be
+		// copied, as append() may sometimes reuse and modify the array
+		ns.visitedNodes = append(ns.visitedNodes, &nodeContext)
+		ns.DebugVisitedNodes()
+		return ns.visit(nodeContext)
+	}
+	log.Debug("nope, skip it")
+	return nil
+}
+
+func (ns *NavigationStrategyImpl) DebugVisitedNodes() {
+	log.Debug("Visited Nodes:")
+	for _, candidate := range ns.visitedNodes {
+		log.Debug(" - %v", pathStackToString(candidate.PathStack))
+	}
+}
+
+func (ns *NavigationStrategyImpl) alreadyVisited(pathStack []interface{}) bool {
+	log.Debug("checking already visited pathStack: %v", pathStackToString(pathStack))
+	for _, candidate := range ns.visitedNodes {
+		candidatePathStack := candidate.PathStack
+		if patchStacksMatch(candidatePathStack, pathStack) {
+			log.Debug("paths match, already seen it")
+			return true
+		}
+
+	}
+	log.Debug("never seen it before!")
+	return false
+}
+
+func patchStacksMatch(path1 []interface{}, path2 []interface{}) bool {
+	log.Debug("checking against path: %v", pathStackToString(path1))
+
+	if len(path1) != len(path2) {
+		return false
+	}
+	for index, p1Value := range path1 {
+
+		p2Value := path2[index]
+		if p1Value != p2Value {
+			return false
+		}
+	}
+	return true
+
+}

pkg/yqlib/path_parser.go

@@ -1,7 +1,13 @@
 package yqlib
 
+import (
+	"strconv"
+	"strings"
+)
+
 type PathParser interface {
 	ParsePath(path string) []string
+	MatchesNextPathElement(nodeContext NodeContext, nodeKey string) bool
 }
 
 type pathParser struct{}
@@ -10,7 +16,37 @@ func NewPathParser() PathParser {
 	return &pathParser{}
 }
 
+/**
+ * node: node that we may traverse/visit
+ * head: path element expression to match against
+ * tail: remaining path element expressions
+ * pathStack: stack of actual paths we've matched to get to node
+ * nodeKey: actual value of this nodes 'key' or index.
+ */
+func (p *pathParser) MatchesNextPathElement(nodeContext NodeContext, nodeKey string) bool {
+	head := nodeContext.Head
+	if head == "**" || head == "*" {
+		return true
+	}
+	if head == "+" {
+		log.Debug("head is +, nodeKey is %v", nodeKey)
+		var _, err = strconv.ParseInt(nodeKey, 10, 64) // nolint
+		if err == nil {
+			return true
+		}
+	}
+	var prefixMatch = strings.TrimSuffix(head, "*")
+	if prefixMatch != head {
+		log.Debug("prefix match, %v", strings.HasPrefix(nodeKey, prefixMatch))
+		return strings.HasPrefix(nodeKey, prefixMatch)
+	}
+	return nodeKey == head
+}
+
 func (p *pathParser) ParsePath(path string) []string {
+	if path == "" {
+		return []string{}
+	}
 	return p.parsePathAccum([]string{}, path)
 }
 

pkg/yqlib/path_parser_test.go

@@ -3,14 +3,18 @@ package yqlib
 import (
 	"testing"
 
-	"github.com/mikefarah/yq/v2/test"
+	"github.com/mikefarah/yq/v3/test"
 )
 
+var parser = NewPathParser()
+
 var parsePathsTests = []struct {
 	path          string
 	expectedPaths []string
 }{
 	{"a.b", []string{"a", "b"}},
+	{"a.b.**", []string{"a", "b", "**"}},
+	{"a.b.*", []string{"a", "b", "*"}},
 	{"a.b[0]", []string{"a", "b", "0"}},
 	{"a.b.d[+]", []string{"a", "b", "d", "+"}},
 	{"a", []string{"a"}},
@@ -22,8 +26,53 @@ var parsePathsTests = []struct {
 	{"[0]", []string{"0"}},
 }
 
-func TestParsePath(t *testing.T) {
+func TestPathParserParsePath(t *testing.T) {
 	for _, tt := range parsePathsTests {
-		test.AssertResultComplex(t, tt.expectedPaths, NewPathParser().ParsePath(tt.path))
+		test.AssertResultComplex(t, tt.expectedPaths, parser.ParsePath(tt.path))
 	}
 }
+
+func TestPathParserMatchesNextPathElementSplat(t *testing.T) {
+	var node = NodeContext{Head: "*"}
+	test.AssertResult(t, true, parser.MatchesNextPathElement(node, ""))
+}
+
+func TestPathParserMatchesNextPathElementDeepSplat(t *testing.T) {
+	var node = NodeContext{Head: "**"}
+	test.AssertResult(t, true, parser.MatchesNextPathElement(node, ""))
+}
+
+func TestPathParserMatchesNextPathElementAppendArrayValid(t *testing.T) {
+	var node = NodeContext{Head: "+"}
+	test.AssertResult(t, true, parser.MatchesNextPathElement(node, "3"))
+}
+
+func TestPathParserMatchesNextPathElementAppendArrayInvalid(t *testing.T) {
+	var node = NodeContext{Head: "+"}
+	test.AssertResult(t, false, parser.MatchesNextPathElement(node, "cat"))
+}
+
+func TestPathParserMatchesNextPathElementPrefixMatchesWhole(t *testing.T) {
+	var node = NodeContext{Head: "cat*"}
+	test.AssertResult(t, true, parser.MatchesNextPathElement(node, "cat"))
+}
+
+func TestPathParserMatchesNextPathElementPrefixMatchesStart(t *testing.T) {
+	var node = NodeContext{Head: "cat*"}
+	test.AssertResult(t, true, parser.MatchesNextPathElement(node, "caterpillar"))
+}
+
+func TestPathParserMatchesNextPathElementPrefixMismatch(t *testing.T) {
+	var node = NodeContext{Head: "cat*"}
+	test.AssertResult(t, false, parser.MatchesNextPathElement(node, "dog"))
+}
+
+func TestPathParserMatchesNextPathElementExactMatch(t *testing.T) {
+	var node = NodeContext{Head: "farahtek"}
+	test.AssertResult(t, true, parser.MatchesNextPathElement(node, "farahtek"))
+}
+
+func TestPathParserMatchesNextPathElementExactMismatch(t *testing.T) {
+	var node = NodeContext{Head: "farahtek"}
+	test.AssertResult(t, false, parser.MatchesNextPathElement(node, "othertek"))
+}

pkg/yqlib/read_navigation_strategy.go

@@ -0,0 +1,16 @@
+package yqlib
+
+func ReadNavigationStrategy() NavigationStrategy {
+	return &NavigationStrategyImpl{
+		visitedNodes: []*NodeContext{},
+		followAlias: func(nodeContext NodeContext) bool {
+			return true
+		},
+		autoCreateMap: func(nodeContext NodeContext) bool {
+			return false
+		},
+		visit: func(nodeContext NodeContext) error {
+			return nil
+		},
+	}
+}

pkg/yqlib/update_navigation_strategy.go

@@ -0,0 +1,34 @@
+package yqlib
+
+func UpdateNavigationStrategy(updateCommand UpdateCommand, autoCreate bool) NavigationStrategy {
+	return &NavigationStrategyImpl{
+		visitedNodes: []*NodeContext{},
+		followAlias: func(nodeContext NodeContext) bool {
+			return false
+		},
+		autoCreateMap: func(nodeContext NodeContext) bool {
+			return autoCreate
+		},
+		visit: func(nodeContext NodeContext) error {
+			node := nodeContext.Node
+			changesToApply := updateCommand.Value
+			if updateCommand.Overwrite || node.Value == "" {
+				log.Debug("going to update")
+				DebugNode(node)
+				log.Debug("with")
+				DebugNode(changesToApply)
+				node.Value = changesToApply.Value
+				node.Tag = changesToApply.Tag
+				node.Kind = changesToApply.Kind
+				node.Style = changesToApply.Style
+				node.Content = changesToApply.Content
+				node.HeadComment = changesToApply.HeadComment
+				node.LineComment = changesToApply.LineComment
+				node.FootComment = changesToApply.FootComment
+			} else {
+				log.Debug("skipping update as node already has value %v and overwriteFlag is ", node.Value, updateCommand.Overwrite)
+			}
+			return nil
+		},
+	}
+}

pkg/yqlib/value_parser.go

@@ -2,41 +2,46 @@ package yqlib
 
 import (
 	"strconv"
+
+	yaml "gopkg.in/yaml.v3"
 )
 
 type ValueParser interface {
-	ParseValue(argument string) interface{}
+	Parse(argument string, customTag string) *yaml.Node
 }
 
-type valueParser struct{}
+type valueParser struct {
+}
 
 func NewValueParser() ValueParser {
 	return &valueParser{}
 }
 
-func (v *valueParser) ParseValue(argument string) interface{} {
-	var value, err interface{}
-	var inQuotes = len(argument) > 0 && argument[0] == '"'
-	if !inQuotes {
-		intValue, intErr := strconv.ParseInt(argument, 10, 64)
-		floatValue, floatErr := strconv.ParseFloat(argument, 64)
-		if intErr == nil && floatErr == nil {
-			if int64(floatValue) == intValue {
-				return intValue
-			}
-			return floatValue
-		} else if floatErr == nil {
-			// In case cannot parse the int due to large precision
-			return floatValue
-		}
-		value, err = strconv.ParseBool(argument)
+func (v *valueParser) Parse(argument string, customTag string) *yaml.Node {
+	var err interface{}
+	var tag = customTag
+
+	if tag == "" {
+		_, err = strconv.ParseBool(argument)
 		if err == nil {
-			return value
+			tag = "!!bool"
+		}
+		_, err = strconv.ParseFloat(argument, 64)
+		if err == nil {
+			tag = "!!float"
+		}
+		_, err = strconv.ParseInt(argument, 10, 64)
+		if err == nil {
+			tag = "!!int"
+		}
+
+		if argument == "null" {
+			tag = "!!null"
 		}
 		if argument == "[]" {
-			return make([]interface{}, 0)
+			return &yaml.Node{Tag: "!!seq", Kind: yaml.SequenceNode}
 		}
-		return argument
 	}
-	return argument[1 : len(argument)-1]
+	log.Debugf("parsed value '%v', tag: '%v'", argument, tag)
+	return &yaml.Node{Value: argument, Tag: tag, Kind: yaml.ScalarNode}
 }

pkg/yqlib/value_parser_test.go

@@ -3,24 +3,36 @@ package yqlib
 import (
 	"testing"
 
-	"github.com/mikefarah/yq/v2/test"
+	"github.com/mikefarah/yq/v3/test"
+	yaml "gopkg.in/yaml.v3"
 )
 
 var parseValueTests = []struct {
 	argument        string
-	expectedResult  interface{}
+	customTag       string
+	expectedTag     string
 	testDescription string
 }{
-	{"true", true, "boolean"},
-	{"\"true\"", "true", "boolean as string"},
-	{"3.4", 3.4, "number"},
-	{"\"3.4\"", "3.4", "number as string"},
-	{"", "", "empty string"},
-	{"1212121", int64(1212121), "big number"},
+	{"true", "", "!!bool", "boolean"},
+	{"true", "!!str", "!!str", "boolean forced as string"},
+	{"3.4", "", "!!float", "float"},
+	{"1212121", "", "!!int", "big number"},
+	{"1212121.1", "", "!!float", "big float number"},
+	{"3", "", "!!int", "int"},
+	{"null", "", "!!null", "null"},
 }
 
-func TestParseValue(t *testing.T) {
+func TestValueParserParse(t *testing.T) {
 	for _, tt := range parseValueTests {
-		test.AssertResultWithContext(t, tt.expectedResult, NewValueParser().ParseValue(tt.argument), tt.testDescription)
+		actual := NewValueParser().Parse(tt.argument, tt.customTag)
+		test.AssertResultWithContext(t, tt.argument, actual.Value, tt.testDescription)
+		test.AssertResultWithContext(t, tt.expectedTag, actual.Tag, tt.testDescription)
+		test.AssertResult(t, yaml.ScalarNode, actual.Kind)
 	}
 }
+
+func TestValueParserParseEmptyArray(t *testing.T) {
+	actual := NewValueParser().Parse("[]", "")
+	test.AssertResult(t, "!!seq", actual.Tag)
+	test.AssertResult(t, yaml.SequenceNode, actual.Kind)
+}

scripts/coverage.sh

@@ -2,5 +2,5 @@
 
 set -e
 
-go test -coverprofile=coverage.out ./...
-go tool cover -html=coverage.out -o cover/coverage.html
+go test -coverprofile=coverage.out -v $(go list ./... | grep -v -E 'examples' | grep -v -E 'test')
+go tool cover -html=coverage.out -o coverage.html

scripts/test.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-go test -v $(go list ./... | grep -v -E 'examples')
+go test -v $(go list ./... | grep -v -E 'examples' | grep -v -E 'test')

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '2.4.1'
+version: '3.0.0-beta'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.

test.yml

@@ -1,2 +1,3 @@
-a: apple
-c: cat
+b:
+  c: thing
+  d: another thing

test/fixture/keyonly.yaml

@@ -1,6 +0,0 @@
-a:
-  b:
-    c: 1
-  d:
-    e: 2
-  f:

test/utils.go

@@ -9,8 +9,8 @@ import (
 	"strings"
 	"testing"
 
-	yaml "github.com/mikefarah/yaml/v2"
 	"github.com/spf13/cobra"
+	yaml "gopkg.in/yaml.v3"
 )
 
 type resulter struct {
@@ -30,8 +30,8 @@ func RunCmd(c *cobra.Command, input string) resulter {
 	return resulter{err, output, c}
 }
 
-func ParseData(rawData string) yaml.MapSlice {
-	var parsedData yaml.MapSlice
+func ParseData(rawData string) yaml.Node {
+	var parsedData yaml.Node
 	err := yaml.Unmarshal([]byte(rawData), &parsedData)
 	if err != nil {
 		fmt.Printf("Error parsing yaml: %v\n", err)

version.go

@@ -11,12 +11,12 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "2.4.1"
+	Version = "3.0.0"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release
 	// such as "dev" (in development), "beta", "rc1", etc.
-	VersionPrerelease = ""
+	VersionPrerelease = "beta"
 )
 
 // ProductName is the name of the product

yq.go

@@ -6,35 +6,32 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"reflect"
 	"strconv"
 	"strings"
 
-	"github.com/mikefarah/yq/v2/pkg/marshal"
-	"github.com/mikefarah/yq/v2/pkg/yqlib"
+	"github.com/mikefarah/yq/v3/pkg/yqlib"
 
 	errors "github.com/pkg/errors"
 
-	yaml "github.com/mikefarah/yaml/v2"
 	"github.com/spf13/cobra"
 	logging "gopkg.in/op/go-logging.v1"
+	yaml "gopkg.in/yaml.v3"
 )
 
-var trimOutput = true
+var customTag = ""
+var printMode = "v"
 var writeInplace = false
 var writeScript = ""
 var outputToJSON = false
 var overwriteFlag = false
-var keyOnlyFlag = false
+var autoCreateFlag = true
 var allowEmptyFlag = false
 var appendFlag = false
 var verbose = false
 var version = false
 var docIndex = "0"
 var log = logging.MustGetLogger("yq")
-var lib = yqlib.NewYqLib(log)
-var jsonConverter = marshal.NewJsonConverter()
-var yamlConverter = marshal.NewYamlConverter()
+var lib = yqlib.NewYqLib()
 var valueParser = yqlib.NewValueParser()
 
 func main() {
@@ -46,7 +43,6 @@ func main() {
 }
 
 func newCommandCLI() *cobra.Command {
-	yaml.DefaultMapType = reflect.TypeOf(yaml.MapSlice{})
 	var rootCmd = &cobra.Command{
 		Use:   "yq",
 		Short: "yq is a lightweight and portable command-line YAML processor.",
@@ -77,8 +73,8 @@ func newCommandCLI() *cobra.Command {
 		},
 	}
 
-	rootCmd.PersistentFlags().BoolVarP(&trimOutput, "trim", "t", true, "trim yaml output")
 	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose mode")
+	rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json")
 	rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
 
 	rootCmd.AddCommand(
@@ -103,16 +99,16 @@ func createReadCmd() *cobra.Command {
 yq read things.yaml a.b.c
 yq r - a.b.c (reads from stdin)
 yq r things.yaml a.*.c
-yq r -d1 things.yaml a.array[0].blah
-yq r things.yaml a.array[*].blah
-yq r -- things.yaml --key-starting-with-dashes
+yq r things.yaml a.**.c
+yq r -d1 things.yaml 'a.array[0].blah'
+yq r things.yaml 'a.array[*].blah'
+yq r -- things.yaml --key-starting-with-dashes.blah
       `,
 		Long: "Outputs the value of the given path in the yaml file to STDOUT",
 		RunE: readProperty,
 	}
 	cmdRead.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
-	cmdRead.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json")
-	cmdRead.PersistentFlags().BoolVarP(&keyOnlyFlag, "keyonly", "k", false, "output with top level keys only")
+	cmdRead.PersistentFlags().StringVarP(&printMode, "printMode", "p", "v", "print mode (v (values, default), p (paths), pv (path and value pairs)")
 	return cmdRead
 }
 
@@ -122,13 +118,17 @@ func createWriteCmd() *cobra.Command {
 		Aliases: []string{"w"},
 		Short:   "yq w [--inplace/-i] [--script/-s script_file] [--doc/-d index] sample.yaml a.b.c newValue",
 		Example: `
-yq write things.yaml a.b.c cat
+yq write things.yaml a.b.c true
+yq write things.yaml 'a.*.c' true
+yq write things.yaml 'a.**' true
+yq write things.yaml a.b.c --tag '!!str' true
+yq write things.yaml a.b.c --tag '!!float' 3
 yq write --inplace -- things.yaml a.b.c --cat
 yq w -i things.yaml a.b.c cat
 yq w --script update_script.yaml things.yaml
 yq w -i -s update_script.yaml things.yaml
-yq w --doc 2 things.yaml a.b.d[+] foo
-yq w -d2 things.yaml a.b.d[+] foo
+yq w --doc 2 things.yaml 'a.b.d[+]' foo
+yq w -d2 things.yaml 'a.b.d[+]' foo
       `,
 		Long: `Updates the yaml file w.r.t the given path and value.
 Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead.
@@ -136,23 +136,28 @@ Outputs to STDOUT unless the inplace flag is used, in which case the file is upd
 Append value to array adds the value to the end of array.
 
 Update Scripts:
-Note that you can give an update script to perform more sophisticated updated. Update script
-format is a yaml map where the key is the path and the value is..well the value. e.g.:
+Note that you can give an update script to perform more sophisticated update. Update script
+format is list of update commands (update or delete) like so:
 ---
-a.b.c: true,
-a.b.e:
-  - name: bob
+- command: update 
+  path: b.c
+  value:
+    #great 
+    things: frog # wow!
+- command: delete
+  path: b.d
 `,
 		RunE: writeProperty,
 	}
 	cmdWrite.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace")
 	cmdWrite.PersistentFlags().StringVarP(&writeScript, "script", "s", "", "yaml script for updating yaml")
+	cmdWrite.PersistentFlags().StringVarP(&customTag, "tag", "t", "", "set yaml tag (e.g. !!int)")
 	cmdWrite.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
 	return cmdWrite
 }
 
 func createPrefixCmd() *cobra.Command {
-	var cmdWrite = &cobra.Command{
+	var cmdPrefix = &cobra.Command{
 		Use:     "prefix [yaml_file] [path]",
 		Aliases: []string{"p"},
 		Short:   "yq p [--inplace/-i] [--doc/-d index] sample.yaml a.b.c",
@@ -169,9 +174,9 @@ Outputs to STDOUT unless the inplace flag is used, in which case the file is upd
 `,
 		RunE: prefixProperty,
 	}
-	cmdWrite.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace")
-	cmdWrite.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
-	return cmdWrite
+	cmdPrefix.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace")
+	cmdPrefix.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
+	return cmdPrefix
 }
 
 func createDeleteCmd() *cobra.Command {
@@ -181,6 +186,8 @@ func createDeleteCmd() *cobra.Command {
 		Short:   "yq d [--inplace/-i] [--doc/-d index] sample.yaml a.b.c",
 		Example: `
 yq delete things.yaml a.b.c
+yq delete things.yaml a.*.c
+yq delete things.yaml a.**
 yq delete --inplace things.yaml a.b.c
 yq delete --inplace -- things.yaml --key-starting-with-dash
 yq d -i things.yaml a.b.c
@@ -204,6 +211,7 @@ func createNewCmd() *cobra.Command {
 		Example: `
 yq new a.b.c cat
 yq n a.b.c cat
+yq n a.b[+] --tag '!!str' true
 yq n -- --key-starting-with-dash cat
 yq n --script create_script.yaml
       `,
@@ -215,7 +223,8 @@ Note that you can give a create script to perform more sophisticated yaml. This
 `,
 		RunE: newProperty,
 	}
-	cmdNew.PersistentFlags().StringVarP(&writeScript, "script", "s", "", "yaml script for updating yaml")
+	cmdNew.PersistentFlags().StringVarP(&writeScript, "script", "s", "", "yaml script for creating yaml")
+	cmdNew.PersistentFlags().StringVarP(&customTag, "tag", "t", "", "set yaml tag (e.g. !!int)")
 	return cmdNew
 }
 
@@ -231,19 +240,19 @@ yq m -i things.yaml other.yaml
 yq m --overwrite things.yaml other.yaml
 yq m -i -x things.yaml other.yaml
 yq m -i -a things.yaml other.yaml
+yq m -i --autocreate=false things.yaml other.yaml
       `,
 		Long: `Updates the yaml file by adding/updating the path(s) and value(s) from additional yaml file(s).
 Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead.
 
 If overwrite flag is set then existing values will be overwritten using the values from each additional yaml file.
 If append flag is set then existing arrays will be merged with the arrays from each additional yaml file.
-
-Note that if you set both flags only overwrite will take effect.
 `,
 		RunE: mergeProperties,
 	}
 	cmdMerge.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace")
 	cmdMerge.PersistentFlags().BoolVarP(&overwriteFlag, "overwrite", "x", false, "update the yaml file by overwriting existing values")
+	cmdMerge.PersistentFlags().BoolVarP(&autoCreateFlag, "autocreate", "c", true, "automatically create any missing entries")
 	cmdMerge.PersistentFlags().BoolVarP(&appendFlag, "append", "a", false, "update the yaml file by appending array values")
 	cmdMerge.PersistentFlags().BoolVarP(&allowEmptyFlag, "allow-empty", "e", false, "allow empty yaml files")
 	cmdMerge.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
@@ -263,99 +272,117 @@ func readProperty(cmd *cobra.Command, args []string) error {
 	if errorParsingDocIndex != nil {
 		return errorParsingDocIndex
 	}
-	var mappedDocs []interface{}
-	var dataBucket interface{}
-	var currentIndex = 0
-	var errorReadingStream = readStream(args[0], func(decoder *yaml.Decoder) error {
-		for {
-			errorReading := decoder.Decode(&dataBucket)
-			if errorReading == io.EOF {
-				log.Debugf("done %v / %v", currentIndex, docIndexInt)
-				if !updateAll && currentIndex <= docIndexInt {
-					return fmt.Errorf("asked to process document index %v but there are only %v document(s)", docIndex, currentIndex)
-				}
-				return nil
-			}
-			log.Debugf("processing %v - requested index %v", currentIndex, docIndexInt)
-			if updateAll || currentIndex == docIndexInt {
-				log.Debugf("reading %v in index %v", path, currentIndex)
-				if path == "" {
-					log.Debug("no path")
-					log.Debugf("%v", dataBucket)
-					mappedDocs = append(mappedDocs, dataBucket)
-				} else {
-					mappedDoc, errorParsing := lib.ReadPath(dataBucket, path)
-					log.Debugf("%v", mappedDoc)
-					if errorParsing != nil {
-						return errors.Wrapf(errorParsing, "Error reading path in document index %v", currentIndex)
-					}
-					mappedDocs = append(mappedDocs, mappedDoc)
-				}
-			}
-			currentIndex = currentIndex + 1
-		}
-	})
+
+	matchingNodes, errorReadingStream := readYamlFile(args[0], path, updateAll, docIndexInt)
 
 	if errorReadingStream != nil {
 		return errorReadingStream
 	}
 
-	if !updateAll {
-		dataBucket = mappedDocs[0]
-	} else {
-		dataBucket = mappedDocs
-	}
+	return printResults(matchingNodes, cmd)
+}
 
-	if keyOnlyFlag {
-		for _, value := range dataBucket.(yaml.MapSlice) {
-			cmd.Println(value.Key)
+func readYamlFile(filename string, path string, updateAll bool, docIndexInt int) ([]*yqlib.NodeContext, error) {
+	var matchingNodes []*yqlib.NodeContext
+
+	var currentIndex = 0
+	var errorReadingStream = readStream(filename, func(decoder *yaml.Decoder) error {
+		for {
+			var dataBucket yaml.Node
+			errorReading := decoder.Decode(&dataBucket)
+
+			if errorReading == io.EOF {
+				return handleEOF(updateAll, docIndexInt, currentIndex)
+			}
+			var errorParsing error
+			matchingNodes, errorParsing = appendDocument(matchingNodes, dataBucket, path, updateAll, docIndexInt, currentIndex)
+			if errorParsing != nil {
+				return errorParsing
+			}
+			currentIndex = currentIndex + 1
 		}
+	})
+	return matchingNodes, errorReadingStream
+}
+
+func handleEOF(updateAll bool, docIndexInt int, currentIndex int) error {
+	log.Debugf("done %v / %v", currentIndex, docIndexInt)
+	if !updateAll && currentIndex <= docIndexInt {
+		return fmt.Errorf("Could not process document index %v as there are only %v document(s)", docIndex, currentIndex)
+	}
+	return nil
+}
+
+func appendDocument(originalMatchingNodes []*yqlib.NodeContext, dataBucket yaml.Node, path string, updateAll bool, docIndexInt int, currentIndex int) ([]*yqlib.NodeContext, error) {
+	log.Debugf("processing document %v - requested index %v", currentIndex, docIndexInt)
+	yqlib.DebugNode(&dataBucket)
+	if !updateAll && currentIndex != docIndexInt {
+		return originalMatchingNodes, nil
+	}
+	log.Debugf("reading %v in document %v", path, currentIndex)
+	matchingNodes, errorParsing := lib.Get(&dataBucket, path)
+	if errorParsing != nil {
+		return nil, errors.Wrapf(errorParsing, "Error reading path in document index %v", currentIndex)
+	}
+	return append(originalMatchingNodes, matchingNodes...), nil
+}
+
+func printValue(node *yaml.Node, cmd *cobra.Command) error {
+	if node.Kind == yaml.ScalarNode {
+		cmd.Print(node.Value)
 		return nil
 	}
 
-	dataStr, err := toString(dataBucket)
-	if err != nil {
-		return err
-	}
-	cmd.Println(dataStr)
-	return nil
-}
+	bufferedWriter := bufio.NewWriter(cmd.OutOrStdout())
+	defer safelyFlush(bufferedWriter)
 
-func newProperty(cmd *cobra.Command, args []string) error {
-	updatedData, err := newYaml(args)
-	if err != nil {
-		return err
-	}
-	dataStr, err := toString(updatedData)
-	if err != nil {
-		return err
-	}
-	cmd.Println(dataStr)
-	return nil
-}
-
-func newYaml(args []string) (interface{}, error) {
-	var writeCommands, writeCommandsError = readWriteCommands(args, 2, "Must provide <path_to_update> <value>")
-	if writeCommandsError != nil {
-		return nil, writeCommandsError
-	}
-
-	var dataBucket interface{}
-	var isArray = strings.HasPrefix(writeCommands[0].Key.(string), "[")
-	if isArray {
-		dataBucket = make([]interface{}, 0)
+	var encoder yqlib.Encoder
+	if outputToJSON {
+		encoder = yqlib.NewJsonEncoder(bufferedWriter)
 	} else {
-		dataBucket = make(yaml.MapSlice, 0)
+		encoder = yqlib.NewYamlEncoder(bufferedWriter)
+	}
+	if err := encoder.Encode(node); err != nil {
+		return err
+	}
+	return nil
+}
+
+func printResults(matchingNodes []*yqlib.NodeContext, cmd *cobra.Command) error {
+	if len(matchingNodes) == 0 {
+		log.Debug("no matching results, nothing to print")
+		return nil
 	}
 
-	for _, entry := range writeCommands {
-		path := entry.Key.(string)
-		value := entry.Value
-		log.Debugf("setting %v to %v", path, value)
-		dataBucket = lib.WritePath(dataBucket, path, value)
+	for index, mappedDoc := range matchingNodes {
+		switch printMode {
+		case "p":
+			cmd.Print(lib.PathStackToString(mappedDoc.PathStack))
+			if index < len(matchingNodes)-1 {
+				cmd.Print("\n")
+			}
+		case "pv", "vp":
+			// put it into a node and print that.
+			var parentNode = yaml.Node{Kind: yaml.MappingNode}
+			parentNode.Content = make([]*yaml.Node, 2)
+			parentNode.Content[0] = &yaml.Node{Kind: yaml.ScalarNode, Value: lib.PathStackToString(mappedDoc.PathStack)}
+			parentNode.Content[1] = mappedDoc.Node
+			if err := printValue(&parentNode, cmd); err != nil {
+				return err
+			}
+		default:
+			if err := printValue(mappedDoc.Node, cmd); err != nil {
+				return err
+			}
+			// Printing our Scalars does not print a new line at the end
+			// we only want to do that if there are more values (so users can easily script extraction of values in the yaml)
+			if index < len(matchingNodes)-1 && mappedDoc.Node.Kind == yaml.ScalarNode {
+				cmd.Print("\n")
+			}
+		}
 	}
 
-	return dataBucket, nil
+	return nil
 }
 
 func parseDocumentIndex() (bool, int, error) {
@@ -369,11 +396,11 @@ func parseDocumentIndex() (bool, int, error) {
 	return false, int(docIndexInt64), nil
 }
 
-type updateDataFn func(dataBucket interface{}, currentIndex int) (interface{}, error)
+type updateDataFn func(dataBucket *yaml.Node, currentIndex int) error
 
-func mapYamlDecoder(updateData updateDataFn, encoder *yaml.Encoder) yamlDecoderFn {
+func mapYamlDecoder(updateData updateDataFn, encoder yqlib.Encoder) yamlDecoderFn {
 	return func(decoder *yaml.Decoder) error {
-		var dataBucket interface{}
+		var dataBucket yaml.Node
 		var errorReading error
 		var errorWriting error
 		var errorUpdating error
@@ -396,12 +423,12 @@ func mapYamlDecoder(updateData updateDataFn, encoder *yaml.Encoder) yamlDecoderF
 			} else if errorReading != nil {
 				return errors.Wrapf(errorReading, "Error reading document at index %v, %v", currentIndex, errorReading)
 			}
-			dataBucket, errorUpdating = updateData(dataBucket, currentIndex)
+			errorUpdating = updateData(&dataBucket, currentIndex)
 			if errorUpdating != nil {
 				return errors.Wrapf(errorUpdating, "Error updating document at index %v", currentIndex)
 			}
 
-			errorWriting = encoder.Encode(dataBucket)
+			errorWriting = encoder.Encode(&dataBucket)
 
 			if errorWriting != nil {
 				return errors.Wrapf(errorWriting, "Error writing document at index %v, %v", currentIndex, errorWriting)
@@ -412,51 +439,125 @@ func mapYamlDecoder(updateData updateDataFn, encoder *yaml.Encoder) yamlDecoderF
 }
 
 func writeProperty(cmd *cobra.Command, args []string) error {
-	var writeCommands, writeCommandsError = readWriteCommands(args, 3, "Must provide <filename> <path_to_update> <value>")
-	if writeCommandsError != nil {
-		return writeCommandsError
+	var updateCommands, updateCommandsError = readUpdateCommands(args, 3, "Must provide <filename> <path_to_update> <value>")
+	if updateCommandsError != nil {
+		return updateCommandsError
 	}
+	return updateDoc(args[0], updateCommands, cmd.OutOrStdout())
+}
+
+func mergeProperties(cmd *cobra.Command, args []string) error {
+	if len(args) < 2 {
+		return errors.New("Must provide at least 2 yaml files")
+	}
+	// first generate update commands from the file
+	var filesToMerge = args[1:]
+	var updateCommands []yqlib.UpdateCommand = make([]yqlib.UpdateCommand, 0)
+
+	for _, fileToMerge := range filesToMerge {
+		matchingNodes, errorProcessingFile := readYamlFile(fileToMerge, "**", false, 0)
+		if errorProcessingFile != nil && (!allowEmptyFlag || !strings.HasPrefix(errorProcessingFile.Error(), "Could not process document index")) {
+			return errorProcessingFile
+		}
+		for _, matchingNode := range matchingNodes {
+			mergePath := lib.MergePathStackToString(matchingNode.PathStack, appendFlag)
+			updateCommands = append(updateCommands, yqlib.UpdateCommand{Command: "update", Path: mergePath, Value: matchingNode.Node, Overwrite: overwriteFlag})
+		}
+	}
+
+	return updateDoc(args[0], updateCommands, cmd.OutOrStdout())
+}
+
+func newProperty(cmd *cobra.Command, args []string) error {
+	var updateCommands, updateCommandsError = readUpdateCommands(args, 2, "Must provide <path_to_update> <value>")
+	if updateCommandsError != nil {
+		return updateCommandsError
+	}
+	newNode := lib.New(updateCommands[0].Path)
+
+	for _, updateCommand := range updateCommands {
+
+		errorUpdating := lib.Update(&newNode, updateCommand, true)
+
+		if errorUpdating != nil {
+			return errorUpdating
+		}
+	}
+
+	var encoder = yaml.NewEncoder(cmd.OutOrStdout())
+	encoder.SetIndent(2)
+	errorEncoding := encoder.Encode(&newNode)
+	encoder.Close()
+	return errorEncoding
+}
+
+func prefixProperty(cmd *cobra.Command, args []string) error {
+
+	if len(args) < 2 {
+		return errors.New("Must provide <filename> <prefixed_path>")
+	}
+	updateCommand := yqlib.UpdateCommand{Command: "update", Path: args[1]}
+	log.Debugf("args %v", args)
+
 	var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex()
 	if errorParsingDocIndex != nil {
 		return errorParsingDocIndex
 	}
 
-	var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) {
-		if updateAll || currentIndex == docIndexInt {
-			log.Debugf("Updating doc %v", currentIndex)
-			for _, entry := range writeCommands {
-				path := entry.Key.(string)
-				value := entry.Value
-				log.Debugf("setting %v to %v", path, value)
-				dataBucket = lib.WritePath(dataBucket, path, value)
-			}
-		}
-		return dataBucket, nil
+	var updateData = func(dataBucket *yaml.Node, currentIndex int) error {
+		return prefixDocument(updateAll, docIndexInt, currentIndex, dataBucket, updateCommand)
 	}
 	return readAndUpdate(cmd.OutOrStdout(), args[0], updateData)
 }
 
-func prefixProperty(cmd *cobra.Command, args []string) error {
-	if len(args) != 2 {
-		return errors.New("Must provide <filename> <prefixed_path>")
-	}
-	prefixPath := args[1]
+func prefixDocument(updateAll bool, docIndexInt int, currentIndex int, dataBucket *yaml.Node, updateCommand yqlib.UpdateCommand) error {
+	if updateAll || currentIndex == docIndexInt {
+		log.Debugf("Prefixing document %v", currentIndex)
+		yqlib.DebugNode(dataBucket)
+		updateCommand.Value = dataBucket.Content[0]
+		dataBucket.Content = make([]*yaml.Node, 1)
 
+		newNode := lib.New(updateCommand.Path)
+		dataBucket.Content[0] = &newNode
+
+		errorUpdating := lib.Update(dataBucket, updateCommand, true)
+		if errorUpdating != nil {
+			return errorUpdating
+		}
+	}
+	return nil
+}
+
+func deleteProperty(cmd *cobra.Command, args []string) error {
+	if len(args) < 2 {
+		return errors.New("Must provide <filename> <path_to_delete>")
+	}
+	var updateCommands []yqlib.UpdateCommand = make([]yqlib.UpdateCommand, 1)
+	updateCommands[0] = yqlib.UpdateCommand{Command: "delete", Path: args[1]}
+
+	return updateDoc(args[0], updateCommands, cmd.OutOrStdout())
+}
+
+func updateDoc(inputFile string, updateCommands []yqlib.UpdateCommand, writer io.Writer) error {
 	var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex()
 	if errorParsingDocIndex != nil {
 		return errorParsingDocIndex
 	}
 
-	var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) {
-
+	var updateData = func(dataBucket *yaml.Node, currentIndex int) error {
 		if updateAll || currentIndex == docIndexInt {
-			log.Debugf("Prefixing %v to doc %v", prefixPath, currentIndex)
-			var mapDataBucket = lib.PrefixPath(dataBucket, prefixPath)
-			return mapDataBucket, nil
+			log.Debugf("Updating doc %v", currentIndex)
+			for _, updateCommand := range updateCommands {
+				log.Debugf("Processing update to Path %v", updateCommand.Path)
+				errorUpdating := lib.Update(dataBucket, updateCommand, autoCreateFlag)
+				if errorUpdating != nil {
+					return errorUpdating
+				}
+			}
 		}
-		return dataBucket, nil
+		return nil
 	}
-	return readAndUpdate(cmd.OutOrStdout(), args[0], updateData)
+	return readAndUpdate(writer, inputFile, updateData)
 }
 
 func readAndUpdate(stdOut io.Writer, inputFile string, updateData updateDataFn) error {
@@ -482,106 +583,64 @@ func readAndUpdate(stdOut io.Writer, inputFile string, updateData updateDataFn)
 			safelyRenameFile(tempFile.Name(), inputFile)
 		}()
 	} else {
-		var writer = bufio.NewWriter(stdOut)
-		destination = writer
+		destination = stdOut
 		destinationName = "Stdout"
-		defer safelyFlush(writer)
 	}
-	var encoder = yaml.NewEncoder(destination)
+
 	log.Debugf("Writing to %v from %v", destinationName, inputFile)
+
+	bufferedWriter := bufio.NewWriter(destination)
+	defer safelyFlush(bufferedWriter)
+
+	var encoder yqlib.Encoder
+	if outputToJSON {
+		encoder = yqlib.NewJsonEncoder(bufferedWriter)
+	} else {
+		encoder = yqlib.NewYamlEncoder(bufferedWriter)
+	}
 	return readStream(inputFile, mapYamlDecoder(updateData, encoder))
 }
 
-func deleteProperty(cmd *cobra.Command, args []string) error {
-	if len(args) < 2 {
-		return errors.New("Must provide <filename> <path_to_delete>")
-	}
-	var deletePath = args[1]
-	var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex()
-	if errorParsingDocIndex != nil {
-		return errorParsingDocIndex
-	}
-
-	var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) {
-		if updateAll || currentIndex == docIndexInt {
-			log.Debugf("Deleting path in doc %v", currentIndex)
-			return lib.DeletePath(dataBucket, deletePath)
-		}
-		return dataBucket, nil
-	}
-
-	return readAndUpdate(cmd.OutOrStdout(), args[0], updateData)
+type updateCommandParsed struct {
+	Command string
+	Path    string
+	Value   yaml.Node
 }
 
-func mergeProperties(cmd *cobra.Command, args []string) error {
-	if len(args) < 2 {
-		return errors.New("Must provide at least 2 yaml files")
-	}
-	var input = args[0]
-	var filesToMerge = args[1:]
-	var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex()
-	if errorParsingDocIndex != nil {
-		return errorParsingDocIndex
-	}
-
-	var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) {
-		if updateAll || currentIndex == docIndexInt {
-			log.Debugf("Merging doc %v", currentIndex)
-			var mergedData map[interface{}]interface{}
-			// merge only works for maps, so put everything in a temporary
-			// map
-			var mapDataBucket = make(map[interface{}]interface{})
-			mapDataBucket["root"] = dataBucket
-			if err := lib.Merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil {
-				return nil, err
-			}
-			for _, f := range filesToMerge {
-				var fileToMerge interface{}
-				if err := readData(f, 0, &fileToMerge); err != nil {
-					if allowEmptyFlag && err == io.EOF {
-						continue
-					}
-					return nil, err
-				}
-				mapDataBucket["root"] = fileToMerge
-				if err := lib.Merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil {
-					return nil, err
-				}
-			}
-			return mergedData["root"], nil
-		}
-		return dataBucket, nil
-	}
-	yaml.DefaultMapType = reflect.TypeOf(map[interface{}]interface{}{})
-	defer func() { yaml.DefaultMapType = reflect.TypeOf(yaml.MapSlice{}) }()
-	return readAndUpdate(cmd.OutOrStdout(), input, updateData)
-}
-
-func readWriteCommands(args []string, expectedArgs int, badArgsMessage string) (yaml.MapSlice, error) {
-	var writeCommands yaml.MapSlice
+func readUpdateCommands(args []string, expectedArgs int, badArgsMessage string) ([]yqlib.UpdateCommand, error) {
+	var updateCommands []yqlib.UpdateCommand = make([]yqlib.UpdateCommand, 0)
 	if writeScript != "" {
-		if err := readData(writeScript, 0, &writeCommands); err != nil {
+		var parsedCommands = make([]updateCommandParsed, 0)
+
+		err := readData(writeScript, 0, &parsedCommands)
+
+		if err != nil && err != io.EOF {
 			return nil, err
 		}
+
+		log.Debugf("Read write commands file '%v'", parsedCommands)
+		for index := range parsedCommands {
+			parsedCommand := parsedCommands[index]
+			updateCommand := yqlib.UpdateCommand{Command: parsedCommand.Command, Path: parsedCommand.Path, Value: &parsedCommand.Value, Overwrite: true}
+			updateCommands = append(updateCommands, updateCommand)
+		}
+
+		log.Debugf("Read write commands file '%v'", updateCommands)
 	} else if len(args) < expectedArgs {
 		return nil, errors.New(badArgsMessage)
 	} else {
-		writeCommands = make(yaml.MapSlice, 1)
-		writeCommands[0] = yaml.MapItem{Key: args[expectedArgs-2], Value: valueParser.ParseValue(args[expectedArgs-1])}
+		updateCommands = make([]yqlib.UpdateCommand, 1)
+		log.Debug("args %v", args)
+		log.Debug("path %v", args[expectedArgs-2])
+		log.Debug("Value %v", args[expectedArgs-1])
+		updateCommands[0] = yqlib.UpdateCommand{Command: "update", Path: args[expectedArgs-2], Value: valueParser.Parse(args[expectedArgs-1], customTag), Overwrite: true}
 	}
-	return writeCommands, nil
-}
-
-func toString(context interface{}) (string, error) {
-	if outputToJSON {
-		return jsonConverter.JsonToString(context)
-	}
-	return yamlConverter.YamlToString(context, trimOutput)
+	return updateCommands, nil
 }
 
 func safelyRenameFile(from string, to string) {
 	if renameError := os.Rename(from, to); renameError != nil {
-		log.Debugf("Error renaming from %v to %v, attemting to copy contents", from, to)
+		log.Debugf("Error renaming from %v to %v, attempting to copy contents", from, to)
 		log.Debug(renameError.Error())
 		// can't do this rename when running in docker to a file targeted in a mounted volume,
 		// so gracefully degrade to copying the entire contents.
@@ -592,7 +651,6 @@ func safelyRenameFile(from string, to string) {
 			removeErr := os.Remove(from)
 			if removeErr != nil {
 				log.Errorf("failed removing original file: %s", from)
-				log.Error(removeErr.Error())
 			}
 		}
 	}

yq_test.go

@@ -1,94 +1,60 @@
 package main
 
-import (
-	"bytes"
-	"fmt"
-	"runtime"
-	"testing"
+// import (
+// 	"fmt"
+// 	"runtime"
+// 	"testing"
 
-	"github.com/mikefarah/yq/v2/pkg/marshal"
-	"github.com/mikefarah/yq/v2/test"
-	"github.com/spf13/cobra"
-)
+// 	"github.com/mikefarah/yq/v2/pkg/marshal"
+// 	"github.com/mikefarah/yq/v2/test"
+// )
 
-func TestMultilineString(t *testing.T) {
-	testString := `
-	abcd
-	efg`
-	formattedResult, _ := marshal.NewYamlConverter().YamlToString(testString, false)
-	test.AssertResult(t, testString, formattedResult)
-}
+// func TestMultilineString(t *testing.T) {
+// 	testString := `
+// 	abcd
+// 	efg`
+// 	formattedResult, _ := marshal.NewYamlConverter().YamlToString(testString, false)
+// 	test.AssertResult(t, testString, formattedResult)
+// }
 
-func TestNewYaml(t *testing.T) {
-	result, _ := newYaml([]string{"b.c", "3"})
-	formattedResult := fmt.Sprintf("%v", result)
-	test.AssertResult(t,
-		"[{b [{c 3}]}]",
-		formattedResult)
-}
+// func TestNewYaml(t *testing.T) {
+// 	result, _ := newYaml([]string{"b.c", "3"})
+// 	formattedResult := fmt.Sprintf("%v", result)
+// 	test.AssertResult(t,
+// 		"[{b [{c 3}]}]",
+// 		formattedResult)
+// }
 
-func TestNewYamlArray(t *testing.T) {
-	result, _ := newYaml([]string{"[0].cat", "meow"})
-	formattedResult := fmt.Sprintf("%v", result)
-	test.AssertResult(t,
-		"[[{cat meow}]]",
-		formattedResult)
-}
+// func TestNewYamlArray(t *testing.T) {
+// 	result, _ := newYaml([]string{"[0].cat", "meow"})
+// 	formattedResult := fmt.Sprintf("%v", result)
+// 	test.AssertResult(t,
+// 		"[[{cat meow}]]",
+// 		formattedResult)
+// }
 
-func TestNewYamlBigInt(t *testing.T) {
-	result, _ := newYaml([]string{"b", "1212121"})
-	formattedResult := fmt.Sprintf("%v", result)
-	test.AssertResult(t,
-		"[{b 1212121}]",
-		formattedResult)
-}
+// func TestNewYaml_WithScript(t *testing.T) {
+// 	writeScript = "examples/instruction_sample.yaml"
+// 	expectedResult := `b:
+//   c: cat
+//   e:
+//   - name: Mike Farah`
+// 	result, _ := newYaml([]string{""})
+// 	actualResult, _ := marshal.NewYamlConverter().YamlToString(result, true)
+// 	test.AssertResult(t, expectedResult, actualResult)
+// }
 
-func TestNewYaml_WithScript(t *testing.T) {
-	writeScript = "examples/instruction_sample.yaml"
-	expectedResult := `b:
-  c: cat
-  e:
-  - name: Mike Farah`
-	result, _ := newYaml([]string{""})
-	actualResult, _ := marshal.NewYamlConverter().YamlToString(result, true)
-	test.AssertResult(t, expectedResult, actualResult)
-}
-
-func TestNewYaml_WithUnknownScript(t *testing.T) {
-	writeScript = "fake-unknown"
-	_, err := newYaml([]string{""})
-	if err == nil {
-		t.Error("Expected error due to unknown file")
-	}
-	var expectedOutput string
-	if runtime.GOOS == "windows" {
-		expectedOutput = `open fake-unknown: The system cannot find the file specified.`
-	} else {
-		expectedOutput = `open fake-unknown: no such file or directory`
-	}
-	test.AssertResult(t, expectedOutput, err.Error())
-}
-
-func TestReadWithKeyOnly(t *testing.T) {
-	readCmd := createReadCmd()
-	expectedResult := `b
-d
-f
-`
-	actualResult, err := executeTestCommand(readCmd, "test/fixture/keyonly.yaml", "a", "-k")
-	if err != nil {
-		t.Error(err.Error())
-	}
-	test.AssertResult(t, expectedResult, actualResult)
-}
-
-func executeTestCommand(command *cobra.Command, args ...string) (output string, err error) {
-	buf := new(bytes.Buffer)
-
-	command.SetOutput(buf)
-	command.SetArgs(args)
-
-	_, err = command.ExecuteC()
-
-	return buf.String(), err
-}
+// func TestNewYaml_WithUnknownScript(t *testing.T) {
+// 	writeScript = "fake-unknown"
+// 	_, err := newYaml([]string{""})
+// 	if err == nil {
+// 		t.Error("Expected error due to unknown file")
+// 	}
+// 	var expectedOutput string
+// 	if runtime.GOOS == "windows" {
+// 		expectedOutput = `open fake-unknown: The system cannot find the file specified.`
+// 	} else {
+// 		expectedOutput = `open fake-unknown: no such file or directory`
+// 	}
+// 	test.AssertResult(t, expectedOutput, err.Error())
+// }