incrementing version

cmd/constant.go

@@ -2,10 +2,10 @@ package cmd
 
 var unwrapScalar = true
 
-// var writeInplace = false
+var writeInplace = false
 var outputToJSON = false
 
-// var exitStatus = false
+var exitStatus = false
 var forceColor = false
 var forceNoColor = false
 var colorsEnabled = false
@@ -15,4 +15,4 @@ var nullInput = false
 var verbose = false
 var version = false
 
-// var log = logging.MustGetLogger("yq")
+var completedSuccessfully = false

cmd/evaluate_all_command.go

@@ -1,6 +1,8 @@
 package cmd
 
 import (
+	"errors"
+	"fmt"
 	"os"
 
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
@@ -23,6 +25,7 @@ yq es -n '{"a": "b"}'
 	return cmdEvalAll
 }
 func evaluateAll(cmd *cobra.Command, args []string) error {
+	cmd.SilenceUsage = true
 	// 0 args, read std in
 	// 1 arg, null input, process expression
 	// 1 arg, read file in sequence
@@ -39,7 +42,26 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 	if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) {
 		colorsEnabled = true
 	}
+
+	if writeInplace && len(args) < 2 {
+		return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
+	}
+
+	if writeInplace {
+		// only use colors if its forced
+		colorsEnabled = forceColor
+		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[1])
+		out, err = writeInPlaceHandler.CreateTempFile()
+		if err != nil {
+			return err
+		}
+		// need to indirectly call the function so  that completedSuccessfully is
+		// passed when we finish execution as opposed to now
+		defer func() { writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) }()
+	}
+
 	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+
 	allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator()
 	switch len(args) {
 	case 0:
@@ -51,7 +73,7 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 		}
 	case 1:
 		if nullInput {
-			err = allAtOnceEvaluator.EvaluateFiles(args[0], []string{}, printer)
+			err = yqlib.NewStreamEvaluator().EvaluateNew(args[0], printer)
 		} else {
 			err = allAtOnceEvaluator.EvaluateFiles("", []string{args[0]}, printer)
 		}
@@ -59,6 +81,11 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 		err = allAtOnceEvaluator.EvaluateFiles(args[0], args[1:], printer)
 	}
 
-	cmd.SilenceUsage = true
+	completedSuccessfully = err == nil
+
+	if err == nil && exitStatus && !printer.PrintedAnything() {
+		return errors.New("no matches found")
+	}
+
 	return err
 }

cmd/evalute_sequence_command.go

@@ -1,6 +1,8 @@
 package cmd
 
 import (
+	"errors"
+	"fmt"
 	"os"
 
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
@@ -23,6 +25,7 @@ yq es -n '{"a": "b"}'
 	return cmdEvalSequence
 }
 func evaluateSequence(cmd *cobra.Command, args []string) error {
+	cmd.SilenceUsage = true
 	// 0 args, read std in
 	// 1 arg, null input, process expression
 	// 1 arg, read file in sequence
@@ -39,10 +42,27 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 	if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) {
 		colorsEnabled = true
 	}
+
+	if writeInplace && len(args) < 2 {
+		return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
+	}
+
+	if writeInplace {
+		// only use colors if its forced
+		colorsEnabled = forceColor
+		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[1])
+		out, err = writeInPlaceHandler.CreateTempFile()
+		if err != nil {
+			return err
+		}
+		// need to indirectly call the function so  that completedSuccessfully is
+		// passed when we finish execution as opposed to now
+		defer func() { writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) }()
+	}
+
 	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	streamEvaluator := yqlib.NewStreamEvaluator()
-	allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator()
 
 	switch len(args) {
 	case 0:
@@ -54,14 +74,18 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		}
 	case 1:
 		if nullInput {
-			err = allAtOnceEvaluator.EvaluateFiles(args[0], []string{}, printer)
+			err = streamEvaluator.EvaluateNew(args[0], printer)
 		} else {
 			err = streamEvaluator.EvaluateFiles("", []string{args[0]}, printer)
 		}
 	default:
 		err = streamEvaluator.EvaluateFiles(args[0], args[1:], printer)
 	}
+	completedSuccessfully = err == nil
+
+	if err == nil && exitStatus && !printer.PrintedAnything() {
+		return errors.New("no matches found")
+	}
 
-	cmd.SilenceUsage = true
 	return err
 }

cmd/root.go

@@ -46,6 +46,8 @@ func New() *cobra.Command {
 
 	rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output")
 	rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
+	rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace of first yaml file given.")
+	rootCmd.PersistentFlags().BoolVarP(&exitStatus, "exit-status", "e", false, "set exit status if there are no matches or null or false is returned")
 
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.0.0-alpha2"
+	Version = "4.0.0-beta1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

cmd/write.go

@@ -1,61 +0,0 @@
-package cmd
-
-// import (
-// 	"github.com/spf13/cobra"
-// )
-
-// func createWriteCmd() *cobra.Command {
-// 	var cmdWrite = &cobra.Command{
-// 		Use:     "write [yaml_file] [path_expression] [value]",
-// 		Aliases: []string{"w"},
-// 		Short:   "yq w [--inplace/-i] [--script/-s script_file] [--doc/-d index] sample.yaml 'b.e(name==fr*).value' newValue",
-// 		Example: `
-// yq write things.yaml 'a.b.c' true
-// yq write things.yaml 'a.*.c' true
-// yq write things.yaml 'a.**' true
-// yq write things.yaml 'a.(child.subchild==co*).c' true
-// yq write things.yaml 'a.b.c' --tag '!!str' true # force 'true' to be interpreted as a string instead of bool
-// yq write things.yaml 'a.b.c' --tag '!!float' 3
-// yq write --inplace -- things.yaml 'a.b.c' '--cat' # need to use '--' to stop processing arguments as flags
-// yq w -i things.yaml 'a.b.c' cat
-// yq w -i -s update_script.yaml things.yaml
-// yq w things.yaml 'a.b.d[+]' foo # appends a new node to the 'd' array
-// yq w --doc 2 things.yaml 'a.b.d[+]' foo # updates the 3rd document of the yaml file
-//       `,
-// 		Long: `Updates the yaml file w.r.t the given path and value.
-// Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead.
-
-// Append value to array adds the value to the end of array.
-
-// Update Scripts:
-// Note that you can give an update script to perform more sophisticated update. Update script
-// format is list of update commands (update or delete) like so:
-// ---
-// - command: update
-//   path: b.c
-//   value:
-//     #great
-//     things: frog # wow!
-// - command: delete
-//   path: b.d
-// `,
-// 		RunE: writeProperty,
-// 	}
-// 	cmdWrite.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace")
-// 	cmdWrite.PersistentFlags().StringVarP(&writeScript, "script", "s", "", "yaml script for updating yaml")
-// 	cmdWrite.PersistentFlags().StringVarP(&sourceYamlFile, "from", "f", "", "yaml file for updating yaml (as-is)")
-// 	cmdWrite.PersistentFlags().StringVarP(&customTag, "tag", "t", "", "set yaml tag (e.g. !!int)")
-// 	cmdWrite.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
-// 	cmdWrite.PersistentFlags().StringVarP(&customStyle, "style", "", "", "formatting style of the value: single, double, folded, flow, literal, tagged")
-// 	cmdWrite.PersistentFlags().StringVarP(&anchorName, "anchorName", "", "", "anchor name")
-// 	cmdWrite.PersistentFlags().BoolVarP(&makeAlias, "makeAlias", "", false, "create an alias using the value as the anchor name")
-// 	return cmdWrite
-// }
-
-// func writeProperty(cmd *cobra.Command, args []string) error {
-// 	var updateCommands, updateCommandsError = readUpdateCommands(args, 3, "Must provide <filename> <path_to_update> <value>", true)
-// 	if updateCommandsError != nil {
-// 		return updateCommandsError
-// 	}
-// 	return updateDoc(args[0], updateCommands, cmd.OutOrStdout())
-// }

cmd/write_test.go

@@ -1,610 +0,0 @@
-package cmd
-
-// import (
-// 	"fmt"
-// 	"runtime"
-// 	"strings"
-// 	"testing"
-
-// 	"github.com/mikefarah/yq/v3/test"
-// )
-
-// func TestWriteCmd(t *testing.T) {
-// 	content := `b:
-//   c: 3
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.c 7", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c: 7
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteKeepCommentsCmd(t *testing.T) {
-// 	content := `b:
-//   c: 3 # comment
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.c 7", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c: 7 # comment
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteWithTaggedStyleCmd(t *testing.T) {
-// 	content := `b:
-//   c: dog
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.c cat --tag=!!str --style=tagged", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c: !!str cat
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteWithDoubleQuotedStyleCmd(t *testing.T) {
-// 	content := `b:
-//   c: dog
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.c cat --style=double", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c: "cat"
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteUpdateStyleOnlyCmd(t *testing.T) {
-// 	content := `b:
-//   c: dog
-//   d: things
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.* --style=single", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c: 'dog'
-//   d: 'things'
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteUpdateTagOnlyCmd(t *testing.T) {
-// 	content := `b:
-//   c: true
-//   d: false
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.* --tag=!!str", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c: "true"
-//   d: "false"
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteWithSingleQuotedStyleCmd(t *testing.T) {
-// 	content := `b:
-//   c: dog
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.c cat --style=single", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c: 'cat'
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteWithLiteralStyleCmd(t *testing.T) {
-// 	content := `b:
-//   c: dog
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.c cat --style=literal", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c: |-
-//     cat
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteWithFoldedStyleCmd(t *testing.T) {
-// 	content := `b:
-//   c: dog
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.c cat --style=folded", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c: >-
-//     cat
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteEmptyMultiDocCmd(t *testing.T) {
-// 	content := `# this is empty
-// ---
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s c 7", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `c: 7
-
-// # this is empty
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteSurroundingEmptyMultiDocCmd(t *testing.T) {
-// 	content := `---
-// # empty
-// ---
-// cat: frog
-// ---
-
-// # empty
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s -d1 c 7", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `
-
-// # empty
-// ---
-// cat: frog
-// c: 7
-// ---
-
-// # empty
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteFromFileCmd(t *testing.T) {
-// 	content := `b:
-//   c: 3
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	source := `kittens: are cute # sure are!`
-// 	fromFilename := test.WriteTempYamlFile(source)
-// 	defer test.RemoveTempYamlFile(fromFilename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.c -f %s", filename, fromFilename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c:
-//     kittens: are cute # sure are!
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteEmptyCmd(t *testing.T) {
-// 	content := ``
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.c 7", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c: 7
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteAutoCreateCmd(t *testing.T) {
-// 	content := `applications:
-//   - name: app
-//     env:`
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s applications[0].env.hello world", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `applications:
-//   - name: app
-//     env:
-//       hello: world
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteCmdScript(t *testing.T) {
-// 	content := `b:
-//   c: 3
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	updateScript := `- command: update
-//   path: b.c
-//   value: 7`
-// 	scriptFilename := test.WriteTempYamlFile(updateScript)
-// 	defer test.RemoveTempYamlFile(scriptFilename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write --script %s %s", scriptFilename, filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c: 7
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteCmdEmptyScript(t *testing.T) {
-// 	content := `b:
-//   c: 3
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	updateScript := ``
-// 	scriptFilename := test.WriteTempYamlFile(updateScript)
-// 	defer test.RemoveTempYamlFile(scriptFilename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write --script %s %s", scriptFilename, filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c: 3
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteMultiCmd(t *testing.T) {
-// 	content := `b:
-//   c: 3
-// ---
-// apples: great
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s -d 1 apples ok", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c: 3
-// ---
-// apples: ok
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-// func TestWriteInvalidDocumentIndexCmd(t *testing.T) {
-// 	content := `b:
-//   c: 3
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s -df apples ok", filename))
-// 	if result.Error == nil {
-// 		t.Error("Expected command to fail due to invalid path")
-// 	}
-// 	expectedOutput := `Document index f is not a integer or *: strconv.ParseInt: parsing "f": invalid syntax`
-// 	test.AssertResult(t, expectedOutput, result.Error.Error())
-// }
-
-// func TestWriteBadDocumentIndexCmd(t *testing.T) {
-// 	content := `b:
-//   c: 3
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s -d 1 apples ok", filename))
-// 	if result.Error == nil {
-// 		t.Error("Expected command to fail due to invalid path")
-// 	}
-// 	expectedOutput := `asked to process document index 1 but there are only 1 document(s)`
-// 	test.AssertResult(t, expectedOutput, result.Error.Error())
-// }
-// func TestWriteMultiAllCmd(t *testing.T) {
-// 	content := `b:
-//   c: 3
-// ---
-// apples: great
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s -d * apples ok", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c: 3
-// apples: ok
-// ---
-// apples: ok`
-// 	test.AssertResult(t, expectedOutput, strings.Trim(result.Output, "\n "))
-// }
-
-// func TestWriteCmd_EmptyArray(t *testing.T) {
-// 	content := `b: 3`
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s a []", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b: 3
-// a: []
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteCmd_Error(t *testing.T) {
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, "write")
-// 	if result.Error == nil {
-// 		t.Error("Expected command to fail due to missing arg")
-// 	}
-// 	expectedOutput := `Must provide <filename> <path_to_update> <value>`
-// 	test.AssertResult(t, expectedOutput, result.Error.Error())
-// }
-
-// func TestWriteCmd_ErrorUnreadableFile(t *testing.T) {
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, "write fake-unknown a.b 3")
-// 	if result.Error == nil {
-// 		t.Error("Expected command to fail due to unknown file")
-// 	}
-// 	var expectedOutput string
-// 	if runtime.GOOS == "windows" {
-// 		expectedOutput = `open fake-unknown: The system cannot find the file specified.`
-// 	} else {
-// 		expectedOutput = `open fake-unknown: no such file or directory`
-// 	}
-// 	test.AssertResult(t, expectedOutput, result.Error.Error())
-// }
-
-// func TestWriteCmd_Inplace(t *testing.T) {
-// 	content := `b:
-//   c: 3
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write -i %s b.c 7", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	gotOutput := test.ReadTempYamlFile(filename)
-// 	expectedOutput := `b:
-//   c: 7`
-// 	test.AssertResult(t, expectedOutput, strings.Trim(gotOutput, "\n "))
-// }
-
-// func TestWriteCmd_InplaceError(t *testing.T) {
-// 	content := `b: cat
-//   c: 3
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write -i %s b.c 7", filename))
-// 	if result.Error == nil {
-// 		t.Error("Expected Error to occur!")
-// 	}
-// 	gotOutput := test.ReadTempYamlFile(filename)
-// 	test.AssertResult(t, content, gotOutput)
-// }
-
-// func TestWriteCmd_Append(t *testing.T) {
-// 	content := `b:
-//   - foo
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b[+] 7", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   - foo
-//   - 7
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteCmd_AppendInline(t *testing.T) {
-// 	content := `b: [foo]`
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b[+] 7", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b: [foo, 7]
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteCmd_AppendInlinePretty(t *testing.T) {
-// 	content := `b: [foo]`
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s -P b[+] 7", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   - foo
-//   - 7
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteCmd_AppendEmptyArray(t *testing.T) {
-// 	content := `a: 2
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b[+] v", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `a: 2
-// b:
-//   - v
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteCmd_SplatArray(t *testing.T) {
-// 	content := `b:
-// - c: thing
-// - c: another thing
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b[*].c new", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   - c: new
-//   - c: new
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteCmd_SplatMap(t *testing.T) {
-// 	content := `b:
-//   c: thing
-//   d: another thing
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.* new", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c: new
-//   d: new
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }
-
-// func TestWriteCmd_SplatMapEmpty(t *testing.T) {
-// 	content := `b:
-//   c: thing
-//   d: another thing
-// `
-// 	filename := test.WriteTempYamlFile(content)
-// 	defer test.RemoveTempYamlFile(filename)
-
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, fmt.Sprintf("write %s b.c.* new", filename))
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	expectedOutput := `b:
-//   c: {}
-//   d: another thing
-// `
-// 	test.AssertResult(t, expectedOutput, result.Output)
-// }

pkg/yqlib/constants.go

@@ -1,5 +0,0 @@
-package yqlib
-
-import "gopkg.in/op/go-logging.v1"
-
-var log = logging.MustGetLogger("yq-lib")

pkg/yqlib/doc/.gitignore

@@ -0,0 +1 @@
+*.zip

pkg/yqlib/doc/Add.md

@@ -2,6 +2,32 @@ Add behaves differently according to the type of the LHS:
 - arrays: concatenate
 - number scalars: arithmetic addition (soon)
 - string scalars: concatenate (soon)
+
+Use `+=` as append assign for things like increment. `.a += .x` is equivalent to running `.a |= . + .x`.
+
+## Concatenate and assign arrays
+Given a sample.yml file of:
+```yaml
+a:
+  val: thing
+  b:
+    - cat
+    - dog
+```
+then
+```bash
+yq eval '.a.b += ["cow"]' sample.yml
+```
+will output
+```yaml
+a:
+  val: thing
+  b:
+    - cat
+    - dog
+    - cow
+```
+
 ## Concatenate arrays
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Assign.md

@@ -5,6 +5,18 @@ Which will assign the LHS node values to the RHS node values. The RHS expression
 
 ### relative form: `|=`
 This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment.
+## Create yaml file
+Running
+```bash
+yq eval --null-input '(.a.b = "cat") | (.x = "frog")'
+```
+will output
+```yaml
+a:
+  b: cat
+x: frog
+```
+
 ## Update node to be the child value
 Given a sample.yml file of:
 ```yaml
@@ -148,7 +160,7 @@ Given a sample.yml file of:
 ```
 then
 ```bash
-yq eval '.a.b[0] |= "bogs"' sample.yml
+yq eval '.a.b.[0] |= "bogs"' sample.yml
 ```
 will output
 ```yaml

pkg/yqlib/doc/Multiply.md

@@ -2,6 +2,8 @@ Like the multiple operator in `jq`, depending on the operands, this multiply ope
 
 Upcoming versions of `yq` will add support for other types of multiplication (numbers, strings).
 
+To concatenate when merging objects, use the `*+` form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
+
 Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
 
 ## Merging files
@@ -109,6 +111,38 @@ b:
   - 5
 ```
 
+## Merge, appending arrays
+Given a sample.yml file of:
+```yaml
+a:
+  array:
+    - 1
+    - 2
+    - animal: dog
+  value: coconut
+b:
+  array:
+    - 3
+    - 4
+    - animal: cat
+  value: banana
+```
+then
+```bash
+yq eval '.a *+ .b' sample.yml
+```
+will output
+```yaml
+array:
+  - 1
+  - 2
+  - animal: dog
+  - 3
+  - 4
+  - animal: cat
+value: banana
+```
+
 ## Merge to prefix an element
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Path.md

@@ -1,4 +1,7 @@
 The path operator can be used to get the traversal paths of matching nodes in an expression. The path is returned as an array, which if traversed in order will lead to the matching node.
+
+You can get the key/index of matching nodes by using the `path` operator to return the path array then piping that through `.[-1]` to get the last element of that array, the key.
+
 ## Map path
 Given a sample.yml file of:
 ```yaml
@@ -15,6 +18,21 @@ will output
 - b
 ```
 
+## Get map key
+Given a sample.yml file of:
+```yaml
+a:
+  b: cat
+```
+then
+```bash
+yq eval '.a.b | path | .[-1]' sample.yml
+```
+will output
+```yaml
+b
+```
+
 ## Array path
 Given a sample.yml file of:
 ```yaml
@@ -32,6 +50,22 @@ will output
 - 1
 ```
 
+## Get array index
+Given a sample.yml file of:
+```yaml
+a:
+  - cat
+  - dog
+```
+then
+```bash
+yq eval '.a.[] | select(. == "dog") | path | .[-1]' sample.yml
+```
+will output
+```yaml
+1
+```
+
 ## Print path and value
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Sort Keys.md

@@ -0,0 +1,68 @@
+The Sort Keys operator sorts maps by their keys (based on their string value). This operator does not do anything to arrays or scalars (so you can easily recursively apply it to all maps).
+
+Sort is particularly useful for diffing two different yaml documents:
+
+```bash
+yq eval -i 'sortKeys(..)' file1.yml
+yq eval -i 'sortKeys(..)' file2.yml
+diff file1.yml file2.yml
+```
+
+## Sort keys of map
+Given a sample.yml file of:
+```yaml
+c: frog
+a: blah
+b: bing
+```
+then
+```bash
+yq eval 'sortKeys(.)' sample.yml
+```
+will output
+```yaml
+a: blah
+b: bing
+c: frog
+```
+
+## Sort keys recursively
+Note the array elements are left unsorted, but maps inside arrays are sorted
+
+Given a sample.yml file of:
+```yaml
+bParent:
+  c: dog
+  array:
+    - 3
+    - 1
+    - 2
+aParent:
+  z: donkey
+  x:
+    - c: yum
+      b: delish
+    - b: ew
+      a: apple
+```
+then
+```bash
+yq eval 'sortKeys(..)' sample.yml
+```
+will output
+```yaml
+aParent:
+  x:
+    - b: delish
+      c: yum
+    - a: apple
+      b: ew
+  z: donkey
+bParent:
+  array:
+    - 3
+    - 1
+    - 2
+  c: dog
+```
+

pkg/yqlib/doc/Traverse.md

@@ -138,7 +138,7 @@ Given a sample.yml file of:
 ```
 then
 ```bash
-yq eval '[0]' sample.yml
+yq eval '.[0]' sample.yml
 ```
 will output
 ```yaml
@@ -152,7 +152,7 @@ Given a sample.yml file of:
 ```
 then
 ```bash
-yq eval '[2]' sample.yml
+yq eval '.[2]' sample.yml
 ```
 will output
 ```yaml
@@ -166,7 +166,7 @@ a: b
 ```
 then
 ```bash
-yq eval '[0]' sample.yml
+yq eval '.[0]' sample.yml
 ```
 will output
 ```yaml

pkg/yqlib/doc/headers/Add.md

@@ -1,4 +1,6 @@
 Add behaves differently according to the type of the LHS:
 - arrays: concatenate
 - number scalars: arithmetic addition (soon)
-- string scalars: concatenate (soon)
+- string scalars: concatenate (soon)
+
+Use `+=` as append assign for things like increment. `.a += .x` is equivalent to running `.a |= . + .x`.

pkg/yqlib/doc/headers/Multiply.md

@@ -2,6 +2,8 @@ Like the multiple operator in `jq`, depending on the operands, this multiply ope
 
 Upcoming versions of `yq` will add support for other types of multiplication (numbers, strings).
 
+To concatenate when merging objects, use the `*+` form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
+
 Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
 
 ## Merging files

pkg/yqlib/doc/headers/Path.md

@@ -1 +1,3 @@
-The path operator can be used to get the traversal paths of matching nodes in an expression. The path is returned as an array, which if traversed in order will lead to the matching node.
+The path operator can be used to get the traversal paths of matching nodes in an expression. The path is returned as an array, which if traversed in order will lead to the matching node.
+
+You can get the key/index of matching nodes by using the `path` operator to return the path array then piping that through `.[-1]` to get the last element of that array, the key.

pkg/yqlib/doc/headers/Sort Keys.md

@@ -0,0 +1,9 @@
+The Sort Keys operator sorts maps by their keys (based on their string value). This operator does not do anything to arrays or scalars (so you can easily recursively apply it to all maps).
+
+Sort is particularly useful for diffing two different yaml documents:
+
+```bash
+yq eval -i 'sortKeys(..)' file1.yml
+yq eval -i 'sortKeys(..)' file2.yml
+diff file1.yml file2.yml
+```

pkg/yqlib/file_utils.go

@@ -0,0 +1,50 @@
+package yqlib
+
+import (
+	"io"
+	"os"
+)
+
+func safelyRenameFile(from string, to string) {
+	if renameError := os.Rename(from, to); renameError != nil {
+		log.Debugf("Error renaming from %v to %v, attempting to copy contents", from, to)
+		log.Debug(renameError.Error())
+		// can't do this rename when running in docker to a file targeted in a mounted volume,
+		// so gracefully degrade to copying the entire contents.
+		if copyError := copyFileContents(from, to); copyError != nil {
+			log.Errorf("Failed copying from %v to %v", from, to)
+			log.Error(copyError.Error())
+		} else {
+			removeErr := os.Remove(from)
+			if removeErr != nil {
+				log.Errorf("failed removing original file: %s", from)
+			}
+		}
+	}
+}
+
+// thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
+func copyFileContents(src, dst string) (err error) {
+	in, err := os.Open(src) // nolint gosec
+	if err != nil {
+		return err
+	}
+	defer safelyCloseFile(in)
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer safelyCloseFile(out)
+	if _, err = io.Copy(out, in); err != nil {
+		return err
+	}
+	return out.Sync()
+}
+
+func safelyCloseFile(file *os.File) {
+	err := file.Close()
+	if err != nil {
+		log.Error("Error closing file!")
+		log.Error(err.Error())
+	}
+}

pkg/yqlib/lib.go

@@ -9,6 +9,8 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
+var log = logging.MustGetLogger("yq-lib")
+
 type OperationType struct {
 	Type       string
 	NumArgs    uint // number of arguments to the op
@@ -17,12 +19,10 @@ type OperationType struct {
 }
 
 // operators TODO:
-// - write in place
+// - cookbook doc for common things
-// - mergeAppend (merges and appends arrays)
 // - mergeEmpty (sets only if the document is empty, do I do that now?)
 // - compare ??
 // - validate ??
-// - exists
 
 var Or = &OperationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: OrOperator}
 var And = &OperationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: AndOperator}
@@ -30,6 +30,7 @@ var And = &OperationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: AndOp
 var Union = &OperationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: UnionOperator}
 
 var Assign = &OperationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: AssignUpdateOperator}
+var AddAssign = &OperationType{Type: "ADD_ASSIGN", NumArgs: 2, Precedence: 40, Handler: AddAssignOperator}
 
 var AssignAttributes = &OperationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: AssignAttributesOperator}
 var AssignStyle = &OperationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: AssignStyleOperator}
@@ -54,6 +55,7 @@ var GetFileIndex = &OperationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence
 var GetPath = &OperationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50, Handler: GetPathOperator}
 
 var Explode = &OperationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: ExplodeOperator}
+var SortKeys = &OperationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: SortKeysOperator}
 
 var CollectObject = &OperationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: CollectObjectOperator}
 var TraversePath = &OperationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
@@ -70,9 +72,6 @@ var Select = &OperationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler:
 var Has = &OperationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: HasOperator}
 var DeleteChild = &OperationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: DeleteChildOperator}
 
-// var Exists = &OperationType{Type: "Length", NumArgs: 2, Precedence: 35}
-// filters matches if they have the existing path
-
 type Operation struct {
 	OperationType *OperationType
 	Value         interface{}

pkg/yqlib/operator_add.go

@@ -8,6 +8,20 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
+func createSelfAddOp(rhs *PathTreeNode) *PathTreeNode {
+	return &PathTreeNode{Operation: &Operation{OperationType: Add},
+		Lhs: &PathTreeNode{Operation: &Operation{OperationType: SelfReference}},
+		Rhs: rhs}
+}
+
+func AddAssignOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+	assignmentOp := &Operation{OperationType: Assign}
+	assignmentOp.Preferences = &AssignOpPreferences{true}
+
+	assignmentOpNode := &PathTreeNode{Operation: assignmentOp, Lhs: pathNode.Lhs, Rhs: createSelfAddOp(pathNode.Rhs)}
+	return d.GetMatchingNodes(matchingNodes, assignmentOpNode)
+}
+
 func toNodes(candidates *list.List) []*yaml.Node {
 
 	if candidates.Len() == 0 {
@@ -45,7 +59,7 @@ func AddOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathT
 		lhsCandidate := el.Value.(*CandidateNode)
 		lhsNode := UnwrapDoc(lhsCandidate.Node)
 
-		var newBlank = &CandidateNode{
+		target := &CandidateNode{
 			Path:     lhsCandidate.Path,
 			Document: lhsCandidate.Document,
 			Filename: lhsCandidate.Filename,
@@ -56,11 +70,11 @@ func AddOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathT
 		case yaml.MappingNode:
 			return nil, fmt.Errorf("Maps not yet supported for addition")
 		case yaml.SequenceNode:
-			newBlank.Node.Kind = yaml.SequenceNode
+			target.Node.Kind = yaml.SequenceNode
-			newBlank.Node.Style = lhsNode.Style
+			target.Node.Style = lhsNode.Style
-			newBlank.Node.Tag = "!!seq"
+			target.Node.Tag = "!!seq"
-			newBlank.Node.Content = append(lhsNode.Content, toNodes(rhs)...)
+			target.Node.Content = append(lhsNode.Content, toNodes(rhs)...)
-			results.PushBack(newBlank)
+			results.PushBack(target)
 		case yaml.ScalarNode:
 			return nil, fmt.Errorf("Scalars not yet supported for addition")
 		}

pkg/yqlib/operator_add_test.go

@@ -5,6 +5,14 @@ import (
 )
 
 var addOperatorScenarios = []expressionScenario{
+	{
+		description: "Concatenate and assign arrays",
+		document:    `{a: {val: thing, b: [cat,dog]}}`,
+		expression:  ".a.b += [\"cow\"]",
+		expected: []string{
+			"D0, P[], (doc)::{a: {val: thing, b: [cat, dog, cow]}}\n",
+		},
+	},
 	{
 		description: "Concatenate arrays",
 		document:    `{a: [1,2], b: [3,4]}`,

pkg/yqlib/operator_assign.go

@@ -36,6 +36,11 @@ func AssignUpdateOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNo
 			candidate.UpdateFrom(first.Value.(*CandidateNode))
 		}
 	}
+	// // if there was nothing given, perhaps we are creating a new yaml doc
+	// if matchingNodes.Len() == 0 {
+	// 	log.Debug("started with nothing, returning LHS, %v", lhs.Len())
+	// 	return lhs, nil
+	// }
 	return matchingNodes, nil
 }
 

pkg/yqlib/operator_assign_test.go

@@ -5,6 +5,13 @@ import (
 )
 
 var assignOperatorScenarios = []expressionScenario{
+	{
+		description: "Create yaml file",
+		expression:  `(.a.b = "cat") | (.x = "frog")`,
+		expected: []string{
+			"D0, P[], ()::a:\n    b: cat\nx: frog\n",
+		},
+	},
 	{
 		description: "Update node to be the child value",
 		document:    `{a: {b: {g: foof}}}`,
@@ -99,7 +106,7 @@ var assignOperatorScenarios = []expressionScenario{
 		description:           "Update empty object and array",
 		dontFormatInputForDoc: true,
 		document:              `{}`,
-		expression:            `.a.b[0] |= "bogs"`,
+		expression:            `.a.b.[0] |= "bogs"`,
 		expected: []string{
 			"D0, P[], (doc)::{a: {b: [bogs]}}\n",
 		},
@@ -107,7 +114,7 @@ var assignOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: `.a.b[1].c |= "bogs"`,
+		expression: `.a.b.[1].c |= "bogs"`,
 		expected: []string{
 			"D0, P[], (doc)::{a: {b: [null, {c: bogs}]}}\n",
 		},

pkg/yqlib/operator_collect_object.go

@@ -94,7 +94,7 @@ func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.
 
 			newCandidate.Path = nil
 
-			newCandidate, err = multiply(d, newCandidate, splatCandidate)
+			newCandidate, err = multiply(&MultiplyPreferences{AppendArrays: false})(d, newCandidate, splatCandidate)
 			if err != nil {
 				return nil, err
 			}

pkg/yqlib/operator_collect_test.go

@@ -13,6 +13,14 @@ var collectOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[]\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   ``,
+		expression: `[3]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 3\n",
+		},
+	},
 	{
 		description: "Collect single",
 		document:    ``,

pkg/yqlib/operator_multiply.go

@@ -43,39 +43,49 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, pathNode *Pat
 	return results, nil
 }
 
+type MultiplyPreferences struct {
+	AppendArrays bool
+}
+
 func MultiplyOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
 	log.Debugf("-- MultiplyOperator")
-	return crossFunction(d, matchingNodes, pathNode, multiply)
+	return crossFunction(d, matchingNodes, pathNode, multiply(pathNode.Operation.Preferences.(*MultiplyPreferences)))
 }
 
-func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+func multiply(preferences *MultiplyPreferences) func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-	lhs.Node = UnwrapDoc(lhs.Node)
+	return func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-	rhs.Node = UnwrapDoc(rhs.Node)
+		lhs.Node = UnwrapDoc(lhs.Node)
-	log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
+		rhs.Node = UnwrapDoc(rhs.Node)
-	log.Debugf("-          RHS: %v", rhs.Node.Tag)
+		log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
+		log.Debugf("-          RHS: %v", rhs.Node.Tag)
 
-	if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
+		shouldAppendArrays := preferences.AppendArrays
-		(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
+
+		if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
+			(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
+
+			var newBlank = &CandidateNode{
+				Path:     lhs.Path,
+				Document: lhs.Document,
+				Filename: lhs.Filename,
+				Node:     &yaml.Node{},
+			}
+			var newThing, err = mergeObjects(d, newBlank, lhs, false)
+			if err != nil {
+				return nil, err
+			}
+			return mergeObjects(d, newThing, rhs, shouldAppendArrays)
 
-		var newBlank = &CandidateNode{
-			Path:     lhs.Path,
-			Document: lhs.Document,
-			Filename: lhs.Filename,
-			Node:     &yaml.Node{},
 		}
-		var newThing, err = mergeObjects(d, newBlank, lhs)
+		return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
-		if err != nil {
-			return nil, err
-		}
-		return mergeObjects(d, newThing, rhs)
-
 	}
-	return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
 }
 
-func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) (*CandidateNode, error) {
 	var results = list.New()
-	err := recursiveDecent(d, results, nodeToMap(rhs))
+
+	// shouldn't recurse arrays if appending
+	err := recursiveDecent(d, results, nodeToMap(rhs), !shouldAppendArrays)
 	if err != nil {
 		return nil, err
 	}
@@ -86,7 +96,7 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode)
 	}
 
 	for el := results.Front(); el != nil; el = el.Next() {
-		err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode))
+		err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), shouldAppendArrays)
 		if err != nil {
 			return nil, err
 		}
@@ -107,7 +117,8 @@ func createTraversalTree(path []interface{}) *PathTreeNode {
 
 }
 
-func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode) error {
+func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) error {
+
 	log.Debugf("merge - applyAssignment lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs))
 
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
@@ -116,6 +127,8 @@ func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *Candid
 	if rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode {
 		assignmentOp.OperationType = Assign
 		assignmentOp.Preferences = &AssignOpPreferences{false}
+	} else if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
+		assignmentOp.OperationType = AddAssign
 	}
 	rhsOp := &Operation{OperationType: ValueOp, CandidateNode: rhs}
 

pkg/yqlib/operator_multiply_test.go

@@ -13,6 +13,13 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		expression: `{} * {"cat":"dog"}`,
+		expected: []string{
+			"D0, P[], (!!map)::cat: dog\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `{a: {also: me}, b: {also: [1]}}`,
@@ -92,6 +99,22 @@ b:
 			"D0, P[], (!!map)::{a: [3, 4, 5], b: [3, 4, 5]}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{a: [1], b: [2]}`,
+		expression: `.a *+ .b`,
+		expected: []string{
+			"D0, P[a], (!!seq)::[1, 2]\n",
+		},
+	},
+	{
+		description: "Merge, appending arrays",
+		document:    `{a: {array: [1, 2, animal: dog], value: coconut}, b: {array: [3, 4, animal: cat], value: banana}}`,
+		expression:  `.a *+ .b`,
+		expected: []string{
+			"D0, P[a], (!!map)::{array: [1, 2, {animal: dog}, 3, 4, {animal: cat}], value: banana}\n",
+		},
+	},
 	{
 		description: "Merge to prefix an element",
 		document:    `{a: cat, b: dog}`,

pkg/yqlib/operator_path_test.go

@@ -13,6 +13,14 @@ var pathOperatorScenarios = []expressionScenario{
 			"D0, P[a b], (!!seq)::- a\n- b\n",
 		},
 	},
+	{
+		description: "Get map key",
+		document:    `{a: {b: cat}}`,
+		expression:  `.a.b | path | .[-1]`,
+		expected: []string{
+			"D0, P[a b -1], (!!str)::b\n",
+		},
+	},
 	{
 		description: "Array path",
 		document:    `{a: [cat, dog]}`,
@@ -21,6 +29,14 @@ var pathOperatorScenarios = []expressionScenario{
 			"D0, P[a 1], (!!seq)::- a\n- 1\n",
 		},
 	},
+	{
+		description: "Get array index",
+		document:    `{a: [cat, dog]}`,
+		expression:  `.a.[] | select(. == "dog") | path | .[-1]`,
+		expected: []string{
+			"D0, P[a 1 -1], (!!int)::1\n",
+		},
+	},
 	{
 		description: "Print path and value",
 		document:    `{a: [cat, dog, frog]}`,

pkg/yqlib/operator_recursive_descent.go

@@ -3,13 +3,13 @@ package yqlib
 import (
 	"container/list"
 
-	"gopkg.in/yaml.v3"
+	yaml "gopkg.in/yaml.v3"
 )
 
 func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
 	var results = list.New()
 
-	err := recursiveDecent(d, results, matchMap)
+	err := recursiveDecent(d, results, matchMap, true)
 	if err != nil {
 		return nil, err
 	}
@@ -17,7 +17,7 @@ func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNod
 	return results, nil
 }
 
-func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List) error {
+func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List, recurseArray bool) error {
 	for el := matchMap.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
@@ -26,14 +26,15 @@ func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.Li
 		log.Debugf("Recursive Decent, added %v", NodeToString(candidate))
 		results.PushBack(candidate)
 
-		if candidate.Node.Kind != yaml.AliasNode {
+		if candidate.Node.Kind != yaml.AliasNode && len(candidate.Node.Content) > 0 &&
+			(recurseArray || candidate.Node.Kind != yaml.SequenceNode) {
 
 			children, err := Splat(d, nodeToMap(candidate))
 
 			if err != nil {
 				return err
 			}
-			err = recursiveDecent(d, results, children)
+			err = recursiveDecent(d, results, children, recurseArray)
 			if err != nil {
 				return err
 			}

pkg/yqlib/operator_recursive_descent_test.go

@@ -5,6 +5,22 @@ import (
 )
 
 var recursiveDescentOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: `..`,
+		expected: []string{
+			"D0, P[], (!!map)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[]`,
+		expression: `..`,
+		expected: []string{
+			"D0, P[], (!!seq)::[]\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `cat`,

pkg/yqlib/operator_sort_keys.go

@@ -0,0 +1,53 @@
+package yqlib
+
+import (
+	"container/list"
+	"sort"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func SortKeysOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+
+	for el := matchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		rhs, err := d.GetMatchingNodes(nodeToMap(candidate), pathNode.Rhs)
+		if err != nil {
+			return nil, err
+		}
+
+		for childEl := rhs.Front(); childEl != nil; childEl = childEl.Next() {
+			node := UnwrapDoc(childEl.Value.(*CandidateNode).Node)
+			if node.Kind == yaml.MappingNode {
+				sortKeys(node)
+			}
+			if err != nil {
+				return nil, err
+			}
+		}
+
+	}
+	return matchingNodes, nil
+}
+
+func sortKeys(node *yaml.Node) {
+	keys := make([]string, len(node.Content)/2)
+	keyBucket := map[string]*yaml.Node{}
+	valueBucket := map[string]*yaml.Node{}
+	var contents = node.Content
+	for index := 0; index < len(contents); index = index + 2 {
+		key := contents[index]
+		value := contents[index+1]
+		keys[index/2] = key.Value
+		keyBucket[key.Value] = key
+		valueBucket[key.Value] = value
+	}
+	sort.Strings(keys)
+	sortedContent := make([]*yaml.Node, len(node.Content))
+	for index := 0; index < len(keys); index = index + 1 {
+		keyString := keys[index]
+		sortedContent[index*2] = keyBucket[keyString]
+		sortedContent[1+(index*2)] = valueBucket[keyString]
+	}
+	node.Content = sortedContent
+}

pkg/yqlib/operator_sort_keys_test.go

@@ -0,0 +1,32 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var sortKeysOperatorScenarios = []expressionScenario{
+	{
+		description: "Sort keys of map",
+		document:    `{c: frog, a: blah, b: bing}`,
+		expression:  `sortKeys(.)`,
+		expected: []string{
+			"D0, P[], (doc)::{a: blah, b: bing, c: frog}\n",
+		},
+	},
+	{
+		description:    "Sort keys recursively",
+		subdescription: "Note the array elements are left unsorted, but maps inside arrays are sorted",
+		document:       `{bParent: {c: dog, array: [3,1,2]}, aParent: {z: donkey, x: [{c: yum, b: delish}, {b: ew, a: apple}]}}`,
+		expression:     `sortKeys(..)`,
+		expected: []string{
+			"D0, P[], (!!map)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [3, 1, 2], c: dog}}\n",
+		},
+	},
+}
+
+func TestSortKeysOperatorScenarios(t *testing.T) {
+	for _, tt := range sortKeysOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Sort Keys", sortKeysOperatorScenarios)
+}

pkg/yqlib/operator_traverse_path_test.go

@@ -159,7 +159,7 @@ var traversePathOperatorScenarios = []expressionScenario{
 	{
 		description: "Traversing arrays by index",
 		document:    `[1,2,3]`,
-		expression:  `[0]`,
+		expression:  `.[0]`,
 		expected: []string{
 			"D0, P[0], (!!int)::1\n",
 		},
@@ -167,7 +167,7 @@ var traversePathOperatorScenarios = []expressionScenario{
 	{
 		description: "Maps with numeric keys",
 		document:    `{2: cat}`,
-		expression:  `[2]`,
+		expression:  `.[2]`,
 		expected: []string{
 			"D0, P[2], (!!str)::cat\n",
 		},
@@ -175,7 +175,7 @@ var traversePathOperatorScenarios = []expressionScenario{
 	{
 		description: "Maps with non existing numeric keys",
 		document:    `{a: b}`,
-		expression:  `[0]`,
+		expression:  `.[0]`,
 		expected: []string{
 			"D0, P[0], (!!null)::null\n",
 		},

pkg/yqlib/operators_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 
 	"github.com/mikefarah/yq/v4/test"
+	yaml "gopkg.in/yaml.v3"
 )
 
 type expressionScenario struct {
@@ -37,9 +38,18 @@ func testScenario(t *testing.T, s *expressionScenario) {
 	if s.document != "" {
 		inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml", 0)
 		if err != nil {
-			t.Error(err)
+			t.Error(err, s.document)
 			return
 		}
+	} else {
+		candidateNode := &CandidateNode{
+			Document:  0,
+			Filename:  "",
+			Node:      &yaml.Node{Tag: "!!null"},
+			FileIndex: 0,
+		}
+		inputs.PushBack(candidateNode)
+
 	}
 
 	results, err = treeNavigator.GetMatchingNodes(inputs, node)
@@ -152,20 +162,20 @@ func documentScenarios(t *testing.T, title string, scenarios []expressionScenari
 			var output bytes.Buffer
 			var err error
 			printer := NewPrinter(bufio.NewWriter(&output), false, true, false, 2, true)
+			streamEvaluator := NewStreamEvaluator()
 
 			if s.document != "" {
 				node, err := treeCreator.ParsePath(s.expression)
 				if err != nil {
 					t.Error(err)
 				}
-				streamEvaluator := NewStreamEvaluator()
 				err = streamEvaluator.Evaluate("sample.yaml", strings.NewReader(formattedDoc), node, printer)
+
 				if err != nil {
 					t.Error(err)
 				}
 			} else {
-				allAtOnceEvaluator := NewAllAtOnceEvaluator()
+				err = streamEvaluator.EvaluateNew(s.expression, printer)
-				err = allAtOnceEvaluator.EvaluateFiles(s.expression, []string{}, printer)
 				if err != nil {
 					t.Error(err)
 				}

pkg/yqlib/path_parse_test.go

@@ -49,6 +49,11 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "[", "]"),
 		append(make([]interface{}, 0), "EMPTY", "COLLECT", "PIPE"),
 	},
+	{
+		`[3]`,
+		append(make([]interface{}, 0), "[", "3 (int64)", "]"),
+		append(make([]interface{}, 0), "3 (int64)", "COLLECT", "PIPE"),
+	},
 	{
 		`d0.a`,
 		append(make([]interface{}, 0), "d0", "PIPE", "a"),

pkg/yqlib/path_tokeniser.go

@@ -190,6 +190,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`,`), opToken(Union))
 	lexer.Add([]byte(`:\s*`), opToken(CreateMap))
 	lexer.Add([]byte(`length`), opToken(Length))
+	lexer.Add([]byte(`sortKeys`), opToken(SortKeys))
 	lexer.Add([]byte(`select`), opToken(Select))
 	lexer.Add([]byte(`has`), opToken(Has))
 	lexer.Add([]byte(`explode`), opToken(Explode))
@@ -223,7 +224,6 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte(`\s*\|=\s*`), opTokenWithPrefs(Assign, nil, &AssignOpPreferences{true}))
 
-	lexer.Add([]byte(`\[-?[0-9]+\]`), arrayIndextoken(false))
 	lexer.Add([]byte(`\.\[-?[0-9]+\]`), arrayIndextoken(true))
 
 	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
@@ -251,8 +251,10 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`\]`), literalToken(CloseCollect, true))
 	lexer.Add([]byte(`\{`), literalToken(OpenCollectObject, false))
 	lexer.Add([]byte(`\}`), literalToken(CloseCollectObject, true))
-	lexer.Add([]byte(`\*`), opToken(Multiply))
+	lexer.Add([]byte(`\*`), opTokenWithPrefs(Multiply, nil, &MultiplyPreferences{AppendArrays: false}))
+	lexer.Add([]byte(`\*\+`), opTokenWithPrefs(Multiply, nil, &MultiplyPreferences{AppendArrays: true}))
 	lexer.Add([]byte(`\+`), opToken(Add))
+	lexer.Add([]byte(`\+=`), opToken(AddAssign))
 
 	err := lexer.Compile()
 	if err != nil {

pkg/yqlib/printer.go

@@ -5,11 +5,12 @@ import (
 	"container/list"
 	"io"
 
-	"gopkg.in/yaml.v3"
+	yaml "gopkg.in/yaml.v3"
 )
 
 type Printer interface {
 	PrintResults(matchingNodes *list.List) error
+	PrintedAnything() bool
 }
 
 type resultsPrinter struct {
@@ -21,6 +22,7 @@ type resultsPrinter struct {
 	writer             io.Writer
 	firstTimePrinting  bool
 	previousDocIndex   uint
+	printedMatches     bool
 }
 
 func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
@@ -35,7 +37,14 @@ func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEn
 	}
 }
 
+func (p *resultsPrinter) PrintedAnything() bool {
+	return p.printedMatches
+}
+
 func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
+	p.printedMatches = p.printedMatches || (node.Tag != "!!null" &&
+		(node.Tag != "!!bool" || node.Value != "false"))
+
 	var encoder Encoder
 	if node.Kind == yaml.ScalarNode && p.unwrapScalar && !p.outputToJSON {
 		return p.writeString(writer, node.Value+"\n")
@@ -53,6 +62,13 @@ func (p *resultsPrinter) writeString(writer io.Writer, txt string) error {
 	return errorWriting
 }
 
+func (p *resultsPrinter) safelyFlush(writer *bufio.Writer) {
+	if err := writer.Flush(); err != nil {
+		log.Error("Error flushing writer!")
+		log.Error(err.Error())
+	}
+}
+
 func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	log.Debug("PrintResults for %v matches", matchingNodes.Len())
 	var err error
@@ -66,7 +82,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	}
 
 	bufferedWriter := bufio.NewWriter(p.writer)
-	defer safelyFlush(bufferedWriter)
+	defer p.safelyFlush(bufferedWriter)
 
 	if matchingNodes.Len() == 0 {
 		log.Debug("no matching results, nothing to print")

pkg/yqlib/stream_evaluator.go

@@ -11,6 +11,7 @@ import (
 type StreamEvaluator interface {
 	Evaluate(filename string, reader io.Reader, node *PathTreeNode, printer Printer) error
 	EvaluateFiles(expression string, filenames []string, printer Printer) error
+	EvaluateNew(expression string, printer Printer) error
 }
 
 type streamEvaluator struct {
@@ -23,6 +24,27 @@ func NewStreamEvaluator() StreamEvaluator {
 	return &streamEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewPathTreeCreator()}
 }
 
+func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error {
+	node, err := treeCreator.ParsePath(expression)
+	if err != nil {
+		return err
+	}
+	candidateNode := &CandidateNode{
+		Document:  0,
+		Filename:  "",
+		Node:      &yaml.Node{Tag: "!!null"},
+		FileIndex: 0,
+	}
+	inputList := list.New()
+	inputList.PushBack(candidateNode)
+
+	matches, errorParsing := treeNavigator.GetMatchingNodes(inputList, node)
+	if errorParsing != nil {
+		return errorParsing
+	}
+	return printer.PrintResults(matches)
+}
+
 func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
 
 	node, err := treeCreator.ParsePath(expression)

pkg/yqlib/utils.go

@@ -9,8 +9,6 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-//TODO: convert to interface + struct
-
 var treeNavigator = NewDataTreeNavigator()
 var treeCreator = NewPathTreeCreator()
 
@@ -52,54 +50,3 @@ func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List
 		currentIndex = currentIndex + 1
 	}
 }
-
-// func safelyRenameFile(from string, to string) {
-// 	if renameError := os.Rename(from, to); renameError != nil {
-// 		log.Debugf("Error renaming from %v to %v, attempting to copy contents", from, to)
-// 		log.Debug(renameError.Error())
-// 		// can't do this rename when running in docker to a file targeted in a mounted volume,
-// 		// so gracefully degrade to copying the entire contents.
-// 		if copyError := copyFileContents(from, to); copyError != nil {
-// 			log.Errorf("Failed copying from %v to %v", from, to)
-// 			log.Error(copyError.Error())
-// 		} else {
-// 			removeErr := os.Remove(from)
-// 			if removeErr != nil {
-// 				log.Errorf("failed removing original file: %s", from)
-// 			}
-// 		}
-// 	}
-// }
-
-// thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
-// func copyFileContents(src, dst string) (err error) {
-// 	in, err := os.Open(src) // nolint gosec
-// 	if err != nil {
-// 		return err
-// 	}
-// 	defer safelyCloseFile(in)
-// 	out, err := os.Create(dst)
-// 	if err != nil {
-// 		return err
-// 	}
-// 	defer safelyCloseFile(out)
-// 	if _, err = io.Copy(out, in); err != nil {
-// 		return err
-// 	}
-// 	return out.Sync()
-// }
-
-func safelyFlush(writer *bufio.Writer) {
-	if err := writer.Flush(); err != nil {
-		log.Error("Error flushing writer!")
-		log.Error(err.Error())
-	}
-
-}
-func safelyCloseFile(file *os.File) {
-	err := file.Close()
-	if err != nil {
-		log.Error("Error closing file!")
-		log.Error(err.Error())
-	}
-}

pkg/yqlib/write_in_place_handler.go

@@ -0,0 +1,59 @@
+package yqlib
+
+import (
+	"io/ioutil"
+	"os"
+)
+
+type WriteInPlaceHandler interface {
+	CreateTempFile() (*os.File, error)
+	FinishWriteInPlace(evaluatedSuccessfully bool)
+}
+
+type writeInPlaceHandler struct {
+	inputFilename string
+	tempFile      *os.File
+}
+
+func NewWriteInPlaceHandler(inputFile string) WriteInPlaceHandler {
+
+	return &writeInPlaceHandler{inputFile, nil}
+}
+
+func (w *writeInPlaceHandler) CreateTempFile() (*os.File, error) {
+	info, err := os.Stat(w.inputFilename)
+	if err != nil {
+		return nil, err
+	}
+	_, err = os.Stat(os.TempDir())
+	if os.IsNotExist(err) {
+		err = os.Mkdir(os.TempDir(), 0700)
+		if err != nil {
+			return nil, err
+		}
+	} else if err != nil {
+		return nil, err
+	}
+
+	file, err := ioutil.TempFile("", "temp")
+	if err != nil {
+		return nil, err
+	}
+
+	err = os.Chmod(file.Name(), info.Mode())
+	log.Debug("writing to tempfile: %v", file.Name())
+	w.tempFile = file
+	return file, err
+}
+
+func (w *writeInPlaceHandler) FinishWriteInPlace(evaluatedSuccessfully bool) {
+	log.Debug("Going to write-inplace, evaluatedSuccessfully=%v, target=%v", evaluatedSuccessfully, w.inputFilename)
+	safelyCloseFile(w.tempFile)
+	if evaluatedSuccessfully {
+		log.Debug("moved temp file to target")
+		safelyRenameFile(w.tempFile.Name(), w.inputFilename)
+	} else {
+		log.Debug("removed temp file")
+		os.Remove(w.tempFile.Name())
+	}
+}

scripts/acceptance.sh

@@ -3,10 +3,62 @@
 set -e
 
 # acceptance test
-X=$(./yq e '.b.c |= 3' ./examples/sample.yaml | ./yq e '.b.c' -)
 
-if [[ $X != 3 ]]; then
+
-  echo "Failed acceptance test: expected 3 but was $X"
+
+echo "test eval-sequence"
+random=$((1 + $RANDOM % 10))
+./yq e -n ".a = $random" > test.yml
+X=$(./yq e '.a' test.yml)
+
+if [[ $X != $random ]]; then
+  echo "Failed create: expected $random but was $X"
   exit 1
 fi
+
+echo "--success"
+
+echo "test update-in-place"
+
+update=$(($random + 1))
+./yq e -i ".a = $update" test.yml
+
+X=$(./yq e '.a' test.yml)
+if [[ $X != $update ]]; then
+  echo "Failed to update inplace test: expected $update but was $X"
+  exit 1
+fi
+
+echo "--success"
+
+echo "test eval-all"
+./yq ea -n ".a = $random" > test-eval-all.yml
+Y=$(./yq ea '.a' test-eval-all.yml)
+
+if [[ $Y != $random ]]; then
+  echo "Failed create with eval all: expected $random but was $X"
+  exit 1
+fi
+echo "--success"
+
+echo "test no exit status"
+./yq e '.z' test.yml
+echo "--success"
+
+echo "test exit status"
+set +e
+
+./yq e -e '.z' test.yml
+
+if [[ $? != 1 ]]; then
+  echo "Expected error code 1 but was $?"
+  exit 1
+fi
+
+echo "--success"
+
+set -e
+
+rm test.yml
+rm test-eval-all.yml
 echo "acceptance tests passed"

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.0.0-alpha2'
+version: '4.0.0-beta1'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.