moar tests

commands_test.go

@@ -104,6 +104,39 @@ func TestReadCmd(t *testing.T) {
 	assertResult(t, "2\n", result.Output)
 }
 
+func TestReadInvalidDocumentIndexCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := runCmd(cmd, "read -df examples/sample.yaml b.c")
+	if result.Error == nil {
+		t.Error("Expected command to fail due to invalid path")
+	}
+	expectedOutput := `Document index f is not a integer or *: strconv.ParseInt: parsing "f": invalid syntax`
+	assertResult(t, expectedOutput, result.Error.Error())
+}
+
+func TestReadBadDocumentIndexCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := runCmd(cmd, "read -d1 examples/sample.yaml b.c")
+	if result.Error == nil {
+		t.Error("Expected command to fail due to invalid path")
+	}
+	expectedOutput := `Asked to process document index 1 but there are only 1 document(s)`
+	assertResult(t, expectedOutput, result.Error.Error())
+}
+
+func TestReadOrderCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := runCmd(cmd, "read examples/order.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	assertResult(t,
+		`version: 3
+application: MyApp
+`,
+		result.Output)
+}
+
 func TestReadMultiCmd(t *testing.T) {
 	cmd := getRootCommand()
 	result := runCmd(cmd, "read -d 1 examples/multiple_docs.yaml another.document")
@@ -113,6 +146,19 @@ func TestReadMultiCmd(t *testing.T) {
 	assertResult(t, "here\n", result.Output)
 }
 
+func TestReadMultiAllCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := runCmd(cmd, "read -d* examples/multiple_docs.yaml commonKey")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	assertResult(t,
+		`- first document
+- second document
+- third document
+`, result.Output)
+}
+
 func TestReadCmd_ArrayYaml(t *testing.T) {
 	cmd := getRootCommand()
 	result := runCmd(cmd, "read examples/array.yaml [0].gather_facts")
@@ -192,7 +238,7 @@ func TestReadCmd_ArrayYaml_ErrorBadPath(t *testing.T) {
 	if result.Error == nil {
 		t.Error("Expected command to fail due to invalid path")
 	}
-	expectedOutput := `Error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
+	expectedOutput := `Error reading path in document index 0: Error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
 	assertResult(t, expectedOutput, result.Error.Error())
 }
 
@@ -202,7 +248,7 @@ func TestReadCmd_ArrayYaml_Splat_ErrorBadPath(t *testing.T) {
 	if result.Error == nil {
 		t.Error("Expected command to fail due to invalid path")
 	}
-	expectedOutput := `Error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
+	expectedOutput := `Error reading path in document index 0: Error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
 	assertResult(t, expectedOutput, result.Error.Error())
 }
 
@@ -254,7 +300,7 @@ func TestReadCmd_ErrorBadPath(t *testing.T) {
 	if result.Error == nil {
 		t.Fatal("Expected command to fail due to invalid path")
 	}
-	expectedOutput := `Error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
+	expectedOutput := `Error reading path in document index 0: Error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
 	assertResult(t, expectedOutput, result.Error.Error())
 }
 
@@ -367,7 +413,37 @@ apples: ok
 `
 	assertResult(t, expectedOutput, result.Output)
 }
+func TestWriteInvalidDocumentIndexCmd(t *testing.T) {
+	content := `b:
+  c: 3
+`
+	filename := writeTempYamlFile(content)
+	defer removeTempYamlFile(filename)
 
+	cmd := getRootCommand()
+	result := runCmd(cmd, fmt.Sprintf("write %s -df apples ok", filename))
+	if result.Error == nil {
+		t.Error("Expected command to fail due to invalid path")
+	}
+	expectedOutput := `Document index f is not a integer or *: strconv.ParseInt: parsing "f": invalid syntax`
+	assertResult(t, expectedOutput, result.Error.Error())
+}
+
+func TestWriteBadDocumentIndexCmd(t *testing.T) {
+	content := `b:
+  c: 3
+`
+	filename := writeTempYamlFile(content)
+	defer removeTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := runCmd(cmd, fmt.Sprintf("write %s -d 1 apples ok", filename))
+	if result.Error == nil {
+		t.Error("Expected command to fail due to invalid path")
+	}
+	expectedOutput := `Asked to process document index 1 but there are only 1 document(s)`
+	assertResult(t, expectedOutput, result.Error.Error())
+}
 func TestWriteMultiAllCmd(t *testing.T) {
 	content := `b:
   c: 3
@@ -612,14 +688,46 @@ func TestMergeOverwriteCmd(t *testing.T) {
 	}
 	expectedOutput := `a: other
 b:
-- 1
+- 3
-- 2
+- 4
 c:
   test: 1
 `
 	assertResult(t, expectedOutput, result.Output)
 }
 
+func TestMergeAppendCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := runCmd(cmd, "merge --append examples/data1.yaml examples/data2.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `a: simple
+b:
+- 1
+- 2
+- 3
+- 4
+c:
+  test: 1
+`
+	assertResult(t, expectedOutput, result.Output)
+}
+func TestMergeArraysCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := runCmd(cmd, "merge --append examples/sample_array.yaml examples/sample_array_2.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `- 1
+- 2
+- 3
+- 4
+- 5
+`
+	assertResult(t, expectedOutput, result.Output)
+}
+
 func TestMergeCmd_Multi(t *testing.T) {
 	cmd := getRootCommand()
 	result := runCmd(cmd, "merge -d1 examples/multiple_docs_small.yaml examples/data2.yaml")
@@ -631,6 +739,9 @@ func TestMergeCmd_Multi(t *testing.T) {
 a: other
 another:
   document: here
+b:
+- 3
+- 4
 c:
   test: 1
 ---

examples/data2.yaml

@@ -1,3 +1,4 @@
 a: other
+b: [3, 4]
 c:
   test: 1

examples/multiple_docs.yaml

@@ -16,7 +16,3 @@ another:
 commonKey: third document
 wow: 
   - here is another
----
-- 1
-- 2
-- 3 

examples/sample_array_2.yaml

@@ -0,0 +1 @@
+[4,5]

merge.go

@@ -2,9 +2,11 @@ package main
 
 import "gopkg.in/imdario/mergo.v0"
 
-func merge(dst, src interface{}, overwrite bool) error {
+func merge(dst interface{}, src interface{}, overwrite bool, append bool) error {
 	if overwrite {
-		return mergo.MergeWithOverwrite(dst, src)
+		return mergo.Merge(dst, src, mergo.WithOverride)
+	} else if append {
+		return mergo.Merge(dst, src, mergo.WithAppendSlice)
 	}
 	return mergo.Merge(dst, src)
 }

release_instructions.txt

@@ -1,6 +1,6 @@
 - increment version in version.go
 - increment version in snapcraft.yaml
-- tag git with same version number
+- tag git with same version number, be sure to start with 'v' for gopkg.in
 - make sure local build passes
 - push tag to git
 - make local xcompile (builds binaries for all platforms)

version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "2.0.1"
+	Version = "2.1.0"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

yq.go

@@ -22,6 +22,7 @@ var writeInplace = false
 var writeScript = ""
 var outputToJSON = false
 var overwriteFlag = false
+var appendFlag = false
 var verbose = false
 var version = false
 var docIndex = "0"
@@ -96,7 +97,7 @@ yq r things.yaml a.array[*].blah
 		Long: "Outputs the value of the given path in the yaml file to STDOUT",
 		RunE: readProperty,
 	}
-	cmdRead.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number, 0 based")
+	cmdRead.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
 	cmdRead.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json")
 	return cmdRead
 }
@@ -183,33 +184,83 @@ func createMergeCmd() *cobra.Command {
 	var cmdMerge = &cobra.Command{
 		Use:     "merge [initial_yaml_file] [additional_yaml_file]...",
 		Aliases: []string{"m"},
-		Short:   "yq m [--inplace/-i] [--doc/-d index] [--overwrite/-x] sample.yaml sample2.yaml",
+		Short:   "yq m [--inplace/-i] [--doc/-d index] [--overwrite/-x] [--append/-a] sample.yaml sample2.yaml",
 		Example: `
 yq merge things.yaml other.yaml
 yq merge --inplace things.yaml other.yaml
 yq m -i things.yaml other.yaml
 yq m --overwrite things.yaml other.yaml
 yq m -i -x things.yaml other.yaml
+yq m -i -a things.yaml other.yaml
       `,
 		Long: `Updates the yaml file by adding/updating the path(s) and value(s) from additional yaml file(s).
 Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead.
 
 If overwrite flag is set then existing values will be overwritten using the values from each additional yaml file.
+If append flag is set then existing arrays will be merged with the arrays from each additional yaml file.
+
+Note that if you set both flags only overwrite will take effect.
 `,
 		RunE: mergeProperties,
 	}
 	cmdMerge.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace")
 	cmdMerge.PersistentFlags().BoolVarP(&overwriteFlag, "overwrite", "x", false, "update the yaml file by overwriting existing values")
+	cmdMerge.PersistentFlags().BoolVarP(&appendFlag, "append", "a", false, "update the yaml file by appending array values")
 	cmdMerge.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
 	return cmdMerge
 }
 
 func readProperty(cmd *cobra.Command, args []string) error {
-	data, err := read(args)
+	var path = ""
-	if err != nil {
+
-		return err
+	if len(args) < 1 {
+		return errors.New("Must provide filename")
+	} else if len(args) > 1 {
+		path = args[1]
 	}
-	dataStr, err := toString(data)
+
+	var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex()
+	if errorParsingDocIndex != nil {
+		return errorParsingDocIndex
+	}
+	var mappedDocs []interface{}
+	var dataBucket interface{}
+	var currentIndex = 0
+	var errorReadingStream = readStream(args[0], func(decoder *yaml.Decoder) error {
+		for {
+			errorReading := decoder.Decode(&dataBucket)
+			if errorReading == io.EOF {
+				log.Debugf("done %v / %v", currentIndex, docIndexInt)
+				if !updateAll && currentIndex <= docIndexInt {
+					return fmt.Errorf("Asked to process document index %v but there are only %v document(s)", docIndex, currentIndex)
+				}
+				return nil
+			}
+			log.Debugf("processing %v - requested index %v", currentIndex, docIndexInt)
+			if updateAll || currentIndex == docIndexInt {
+				log.Debugf("reading %v in index %v", path, currentIndex)
+				mappedDoc, errorParsing := readPath(dataBucket, path)
+				log.Debugf("%v", mappedDoc)
+				if errorParsing != nil {
+					return errors.Wrapf(errorParsing, "Error reading path in document index %v", currentIndex)
+				}
+				mappedDocs = append(mappedDocs, mappedDoc)
+			}
+			currentIndex = currentIndex + 1
+		}
+	})
+
+	if errorReadingStream != nil {
+		return errorReadingStream
+	}
+
+	if !updateAll {
+		dataBucket = mappedDocs[0]
+	} else {
+		dataBucket = mappedDocs
+	}
+
+	dataStr, err := toString(dataBucket)
 	if err != nil {
 		return err
 	}
@@ -217,28 +268,13 @@ func readProperty(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-func read(args []string) (interface{}, error) {
+func readPath(dataBucket interface{}, path string) (interface{}, error) {
-	var path = ""
-
-	if len(args) < 1 {
-		return nil, errors.New("Must provide filename")
-	} else if len(args) > 1 {
-		path = args[1]
-	}
-	var generalData interface{}
-	var docIndexInt, errorParsingDocumentIndex = strconv.ParseInt(docIndex, 10, 32)
-	if errorParsingDocumentIndex != nil {
-		return nil, errors.Wrapf(errorParsingDocumentIndex, "Document index %v is not a integer", docIndex)
-	}
-	if err := readData(args[0], int(docIndexInt), &generalData); err != nil {
-		return nil, err
-	}
 	if path == "" {
-		return generalData, nil
+		log.Debug("no path")
+		return dataBucket, nil
 	}
-
 	var paths = parsePath(path)
-	return recurse(generalData, paths[0], paths[1:])
+	return recurse(dataBucket, paths[0], paths[1:])
 }
 
 func newProperty(cmd *cobra.Command, args []string) error {
@@ -310,8 +346,8 @@ func mapYamlDecoder(updateData updateDataFn, encoder *yaml.Encoder) yamlDecoderF
 			errorReading = decoder.Decode(&dataBucket)
 
 			if errorReading == io.EOF {
-				if !updateAll && currentIndex < docIndexInt {
+				if !updateAll && currentIndex <= docIndexInt {
-					return fmt.Errorf("Asked to process document %v but there are only %v document(s)", docIndex, currentIndex)
+					return fmt.Errorf("Asked to process document index %v but there are only %v document(s)", docIndex, currentIndex)
 				}
 				return nil
 			} else if errorReading != nil {
@@ -420,7 +456,11 @@ func mergeProperties(cmd *cobra.Command, args []string) error {
 		if updateAll || currentIndex == docIndexInt {
 			log.Debugf("Merging doc %v", currentIndex)
 			var mergedData map[interface{}]interface{}
-			if err := merge(&mergedData, dataBucket, overwriteFlag); err != nil {
+			// merge only works for maps, so put everything in a temporary
+			// map
+			var mapDataBucket = make(map[interface{}]interface{})
+			mapDataBucket["root"] = dataBucket
+			if err := merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil {
 				return nil, err
 			}
 			for _, f := range filesToMerge {
@@ -428,11 +468,12 @@ func mergeProperties(cmd *cobra.Command, args []string) error {
 				if err := readData(f, 0, &fileToMerge); err != nil {
 					return nil, err
 				}
-				if err := merge(&mergedData, fileToMerge, overwriteFlag); err != nil {
+				mapDataBucket["root"] = fileToMerge
+				if err := merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil {
 					return nil, err
 				}
 			}
-			return mergedData, nil
+			return mergedData["root"], nil
 		}
 		return dataBucket, nil
 	}

yq_test.go

@@ -23,37 +23,6 @@ func TestParseValue(t *testing.T) {
 	}
 }
 
-func TestRead(t *testing.T) {
-	result, _ := read([]string{"examples/sample.yaml", "b.c"})
-	assertResult(t, 2, result)
-}
-
-func TestReadMulti(t *testing.T) {
-	docIndex = "1"
-	result, _ := read([]string{"examples/multiple_docs.yaml", "another.document"})
-	assertResult(t, "here", result)
-	docIndex = "0"
-}
-
-func TestReadArray(t *testing.T) {
-	result, _ := read([]string{"examples/sample_array.yaml", "[1]"})
-	assertResult(t, 2, result)
-}
-
-func TestReadString(t *testing.T) {
-	result, _ := read([]string{"examples/sample_text.yaml"})
-	assertResult(t, "hi", result)
-}
-
-func TestOrder(t *testing.T) {
-	result, _ := read([]string{"examples/order.yaml"})
-	formattedResult, _ := yamlToString(result)
-	assertResult(t,
-		`version: 3
-application: MyApp`,
-		formattedResult)
-}
-
 func TestMultilineString(t *testing.T) {
 	testString := `
 	abcd