Updated readme

.travis.yml

@@ -0,0 +1,4 @@
+language: go
+go:
+    - 1.7.x
+script: ./ci.sh

LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2017 Mike Farah
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,4 +1,4 @@
-# yaml
+# yaml [![Build Status](https://travis-ci.org/mikefarah/yaml.svg?branch=master)](https://travis-ci.org/mikefarah/yaml)
 yaml is a lightweight and flexible command-line YAML processor
 
 The aim of the project is to be the [jq](https://github.com/stedolan/jq) or sed of yaml files.
@@ -14,8 +14,12 @@ go get github.com/mikefarah/yaml
 - Deep read a yaml file with a given path
 - Update a yaml file given a path
 - Update a yaml file given a script file
+- Update creates any missing entries in the path on the fly
+- Create a yaml file given a deep path and value
+- Create a yaml file given a script file
 - Convert from json to yaml
 - Convert from yaml to json
+- Pipe data in by using '-'
 
 ## Read examples
 ```
@@ -111,6 +115,7 @@ will output:
 ```
 
 ## Update examples
+Existing yaml files can be updated via the write command
 
 ### Update to stdout
 Given a sample.yaml file of:
@@ -128,6 +133,31 @@ b:
   c: cat
 ```
 
+### Update from STDIN
+```bash
+cat sample.yaml | yaml w - b.c blah
+```
+
+### Adding new fields
+Any missing fields in the path will be created on the fly.
+
+Given a sample.yaml file of:
+```yaml
+b:
+  c: 2
+```
+then
+```bash
+yaml w sample.yaml b.d[0] "new thing"
+```
+will output:
+```yaml
+b:
+  c: cat
+  d:
+    - new thing
+```
+
 ### Updating yaml in-place
 Given a sample.yaml file of:
 ```yaml
@@ -167,6 +197,51 @@ b:
     - name: Howdy Partner
 ```
 
+And, of course, you can pipe the instructions in using '-':
+```bash
+cat update_instructions.yaml | yaml w -s - sample.yaml
+```
+
+## New Examples
+Yaml files can be created using the 'new' command. This works in the same way as the write command, but you don't pass in an existing Yaml file.
+
+### Creating a simple yaml file
+```bash
+yaml n b.c cat
+```
+will output:
+```yaml
+b:
+  c: cat
+```
+
+### Creating using a create script
+Create scripts follow the same format as the update scripts.
+
+Given a script create_instructions.yaml of:
+```yaml
+b.c: 3
+b.e[0].name: Howdy Partner
+```
+then
+
+```bash
+yaml n -s create_instructions.yaml
+```
+will output:
+```yaml
+b:
+  c: 3
+  e:
+    - name: Howdy Partner
+```
+
+You can also pipe the instructions in:
+
+```bash
+cat create_instructions.yaml | yaml n -s -
+```
+
 ## Converting to and from json
 
 ### Yaml2json

ci.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+go test
+go build
+
+# acceptance test
+X=$(./yaml w sample.yaml b.c 3 | ./yaml r - b.c)
+
+if [ $X != 3 ]
+  then
+  echo "Failed acceptance test: expected 2 but was $X"
+  exit 1
+fi

data_navigator.go

@@ -1,52 +1,118 @@
 package main
 
 import (
-	// "fmt"
+	"github.com/mikefarah/yaml/Godeps/_workspace/src/gopkg.in/yaml.v2"
 	"strconv"
 )
 
-func write(context map[interface{}]interface{}, head string, tail []string, value interface{}) {
-	if len(tail) == 0 {
-		context[head] = value
-	} else {
-		// e.g. if updating a.b.c, we need to get the 'b', this could be a map or an array
-		var parent = readMap(context, head, tail[0:len(tail)-1])
-		switch parent.(type) {
-		case map[interface{}]interface{}:
-			toUpdate := parent.(map[interface{}]interface{})
-			// b is a map, update the key 'c' to the supplied value
-			key := (tail[len(tail)-1])
-			toUpdate[key] = value
-		case []interface{}:
-			toUpdate := parent.([]interface{})
-			// b is an array, update it at index 'c' to the supplied value
-			rawIndex := (tail[len(tail)-1])
-			index, err := strconv.ParseInt(rawIndex, 10, 64)
-			if err != nil {
-				die("Error accessing array: %v", err)
-			}
-			toUpdate[index] = value
+func entryInSlice(context yaml.MapSlice, key interface{}) *yaml.MapItem {
+	for idx := range context {
+		var entry = &context[idx]
+		if entry.Key == key {
+			return entry
 		}
-
 	}
+	return nil
 }
 
-func readMap(context map[interface{}]interface{}, head string, tail []string) interface{} {
+func writeMap(context interface{}, paths []string, value interface{}) yaml.MapSlice {
+	log.Debugf("writeMap for %v for %v with value %v\n", paths, context, value)
+
+	var mapSlice yaml.MapSlice
+	switch context.(type) {
+	case yaml.MapSlice:
+		mapSlice = context.(yaml.MapSlice)
+	default:
+		mapSlice = make(yaml.MapSlice, 0)
+	}
+
+	if len(paths) == 0 {
+		return mapSlice
+	}
+
+	child := entryInSlice(mapSlice, paths[0])
+	if child == nil {
+		newChild := yaml.MapItem{Key: paths[0]}
+		mapSlice = append(mapSlice, newChild)
+		child = entryInSlice(mapSlice, paths[0])
+		log.Debugf("\tAppended child at %v for mapSlice %v\n", paths[0], mapSlice)
+	}
+
+	log.Debugf("\tchild.Value %v\n", child.Value)
+
+	remainingPaths := paths[1:len(paths)]
+	child.Value = updatedChildValue(child.Value, remainingPaths, value)
+	log.Debugf("\tReturning mapSlice %v\n", mapSlice)
+	return mapSlice
+}
+
+func updatedChildValue(child interface{}, remainingPaths []string, value interface{}) interface{} {
+	if len(remainingPaths) == 0 {
+		return value
+	}
+
+	_, nextIndexErr := strconv.ParseInt(remainingPaths[0], 10, 64)
+	if nextIndexErr != nil {
+		// must be a map
+		return writeMap(child, remainingPaths, value)
+	}
+
+	// must be an array
+	return writeArray(child, remainingPaths, value)
+}
+
+func writeArray(context interface{}, paths []string, value interface{}) []interface{} {
+	log.Debugf("writeArray for %v for %v with value %v\n", paths, context, value)
+	var array []interface{}
+	switch context.(type) {
+	case []interface{}:
+		array = context.([]interface{})
+	default:
+		array = make([]interface{}, 1)
+	}
+
+	if len(paths) == 0 {
+		return array
+	}
+
+	log.Debugf("\tarray %v\n", array)
+
+	rawIndex := paths[0]
+	index, err := strconv.ParseInt(rawIndex, 10, 64)
+	if err != nil {
+		die("Error accessing array: %v", err)
+	}
+	currentChild := array[index]
+
+	log.Debugf("\tcurrentChild %v\n", currentChild)
+
+	remainingPaths := paths[1:len(paths)]
+	array[index] = updatedChildValue(currentChild, remainingPaths, value)
+	log.Debugf("\tReturning array %v\n", array)
+	return array
+}
+
+func readMap(context yaml.MapSlice, head string, tail []string) interface{} {
 	if head == "*" {
 		return readMapSplat(context, tail)
 	}
-	value := context[head]
+	var value interface{}
+
+	entry := entryInSlice(context, head)
+	if entry != nil {
+		value = entry.Value
+	}
 	return calculateValue(value, tail)
 }
 
-func readMapSplat(context map[interface{}]interface{}, tail []string) interface{} {
+func readMapSplat(context yaml.MapSlice, tail []string) interface{} {
 	var newArray = make([]interface{}, len(context))
 	var i = 0
-	for _, value := range context {
+	for _, entry := range context {
 		if len(tail) > 0 {
-			newArray[i] = recurse(value, tail[0], tail[1:len(tail)])
+			newArray[i] = recurse(entry.Value, tail[0], tail[1:len(tail)])
 		} else {
-			newArray[i] = value
+			newArray[i] = entry.Value
 		}
 		i++
 	}
@@ -64,8 +130,8 @@ func recurse(value interface{}, head string, tail []string) interface{} {
 			die("Error accessing array: %v", err)
 		}
 		return readArray(value.([]interface{}), index, tail)
-	case map[interface{}]interface{}:
-		return readMap(value.(map[interface{}]interface{}), head, tail)
+	case yaml.MapSlice:
+		return readMap(value.(yaml.MapSlice), head, tail)
 	default:
 		return nil
 	}

data_navigator_test.go

@@ -2,10 +2,19 @@ package main
 
 import (
 	"fmt"
+	"github.com/mikefarah/yaml/Godeps/_workspace/src/gopkg.in/yaml.v2"
+	"github.com/op/go-logging"
+	"os"
 	"sort"
 	"testing"
 )
 
+func TestMain(m *testing.M) {
+	backend.SetLevel(logging.ERROR, "")
+	logging.SetBackend(backend)
+	os.Exit(m.Run())
+}
+
 func TestReadMap_simple(t *testing.T) {
 	var data = parseData(`
 ---
@@ -107,16 +116,48 @@ e:
 	assertResult(t, "[Fred Sam]", fmt.Sprintf("%v", readMap(data, "e", []string{"*", "name"})))
 }
 
+func TestWrite_really_simple(t *testing.T) {
+	var data = parseData(`
+    b: 2
+`)
+
+	updated := writeMap(data, []string{"b"}, "4")
+	b := entryInSlice(updated, "b").Value
+	assertResult(t, "4", b)
+}
+
 func TestWrite_simple(t *testing.T) {
 	var data = parseData(`
 b:
   c: 2
 `)
 
-	write(data, "b", []string{"c"}, "4")
+	updated := writeMap(data, []string{"b", "c"}, "4")
+	b := entryInSlice(updated, "b").Value.(yaml.MapSlice)
+	c := entryInSlice(b, "c").Value
+	assertResult(t, "4", c)
+}
 
-	b := data["b"].(map[interface{}]interface{})
-	assertResult(t, "4", b["c"].(string))
+func TestWrite_new(t *testing.T) {
+	var data = parseData(`
+b:
+  c: 2
+`)
+
+	updated := writeMap(data, []string{"b", "d"}, "4")
+	b := entryInSlice(updated, "b").Value.(yaml.MapSlice)
+	d := entryInSlice(b, "d").Value
+	assertResult(t, "4", d)
+}
+
+func TestWrite_new_deep(t *testing.T) {
+	var data = parseData(`
+b:
+  c: 2
+`)
+
+	updated := writeMap(data, []string{"b", "d", "f"}, "4")
+	assertResult(t, "4", readMap(updated, "b", []string{"d", "f"}))
 }
 
 func TestWrite_array(t *testing.T) {
@@ -125,19 +166,52 @@ b:
   - aa
 `)
 
-	write(data, "b", []string{"0"}, "bb")
+	updated := writeMap(data, []string{"b", "0"}, "bb")
 
-	b := data["b"].([]interface{})
+	b := entryInSlice(updated, "b").Value.([]interface{})
 	assertResult(t, "bb", b[0].(string))
 }
 
+func TestWrite_new_array(t *testing.T) {
+	var data = parseData(`
+b:
+  c: 2
+`)
+
+	updated := writeMap(data, []string{"b", "0"}, "4")
+	assertResult(t, "4", readMap(updated, "b", []string{"0"}))
+}
+
+func TestWrite_new_array_deep(t *testing.T) {
+	var data = parseData(`
+b:
+  c: 2
+`)
+
+	var expected = `b:
+- c: "4"`
+
+	updated := writeMap(data, []string{"b", "0", "c"}, "4")
+	assertResult(t, expected, yamlToString(updated))
+}
+
+func TestWrite_new_map_array_deep(t *testing.T) {
+	var data = parseData(`
+b:
+  c: 2
+`)
+
+	updated := writeMap(data, []string{"b", "d", "0"}, "4")
+	assertResult(t, "4", readMap(updated, "b", []string{"d", "0"}))
+}
+
 func TestWrite_with_no_tail(t *testing.T) {
 	var data = parseData(`
 b:
   c: 2
 `)
-	write(data, "b", []string{}, "4")
+	updated := writeMap(data, []string{"b"}, "4")
 
-	b := data["b"]
+	b := entryInSlice(updated, "b").Value
 	assertResult(t, "4", fmt.Sprintf("%v", b))
 }

json_converter.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/json"
+	"github.com/mikefarah/yaml/Godeps/_workspace/src/gopkg.in/yaml.v2"
 )
 
 func fromJSONBytes(jsonBytes []byte, parsedData *map[interface{}]interface{}) {
@@ -55,11 +56,11 @@ func toJSON(context interface{}) interface{} {
 			newArray[index] = toJSON(value)
 		}
 		return newArray
-	case map[interface{}]interface{}:
-		oldMap := context.(map[interface{}]interface{})
+	case yaml.MapSlice:
+		oldMap := context.(yaml.MapSlice)
 		newMap := make(map[string]interface{})
-		for key, value := range oldMap {
-			newMap[key.(string)] = toJSON(value)
+		for _, entry := range oldMap {
+			newMap[entry.Key.(string)] = toJSON(entry.Value)
 		}
 		return newMap
 	default:

order.yaml

@@ -0,0 +1,2 @@
+version: 3
+application: MyApp

order.yml

@@ -0,0 +1,6 @@
+version: '2'
+services:
+  test:
+    image: ubuntu:14.04
+    stdin_open: true
+    tty: true

precheckin.sh

@@ -1,17 +1,9 @@
 #!/bin/bash
 
+set -e
+
 gofmt -w .
 golint
-go test
-go build
-
-# acceptance test
-X=$(./yaml w sample.yaml b.c 3 | ./yaml r - b.c)
-
-if [ $X != 3 ]
-  then
-	echo "Failed acceptance test: expected 2 but was $X"
-  exit 1
-fi
+./ci.sh
 
 go install

utils_test.go

@@ -7,8 +7,8 @@ import (
 	"testing"
 )
 
-func parseData(rawData string) map[interface{}]interface{} {
-	var parsedData map[interface{}]interface{}
+func parseData(rawData string) yaml.MapSlice {
+	var parsedData yaml.MapSlice
 	err := yaml.Unmarshal([]byte(rawData), &parsedData)
 	if err != nil {
 		fmt.Println("Error parsing yaml: %v", err)

yaml.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/mikefarah/yaml/Godeps/_workspace/src/github.com/spf13/cobra"
 	"github.com/mikefarah/yaml/Godeps/_workspace/src/gopkg.in/yaml.v2"
+	"github.com/op/go-logging"
 	"io/ioutil"
 	"os"
 	"strconv"
@@ -15,17 +16,28 @@ var writeInplace = false
 var writeScript = ""
 var inputJSON = false
 var outputToJSON = false
+var verbose = false
+var log = logging.MustGetLogger("yaml")
+var format = logging.MustStringFormatter(
+	`%{color}%{time:15:04:05} %{shortfunc} [%{level:.4s}]%{color:reset} %{message}`,
+)
+var backend = logging.AddModuleLevel(
+	logging.NewBackendFormatter(logging.NewLogBackend(os.Stderr, "", 0), format))
 
 func main() {
+	backend.SetLevel(logging.ERROR, "")
+	logging.SetBackend(backend)
+
 	var cmdRead = createReadCmd()
 	var cmdWrite = createWriteCmd()
+	var cmdNew = createNewCmd()
 
 	var rootCmd = &cobra.Command{Use: "yaml"}
 	rootCmd.PersistentFlags().BoolVarP(&trimOutput, "trim", "t", true, "trim yaml output")
 	rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json")
 	rootCmd.PersistentFlags().BoolVarP(&inputJSON, "fromjson", "J", false, "input as json")
-
-	rootCmd.AddCommand(cmdRead, cmdWrite)
+	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose mode")
+	rootCmd.AddCommand(cmdRead, cmdWrite, cmdNew)
 	rootCmd.Execute()
 }
 
@@ -76,12 +88,37 @@ a.b.e:
 	return cmdWrite
 }
 
+func createNewCmd() *cobra.Command {
+	var cmdNew = &cobra.Command{
+		Use:     "new [path] [value]",
+		Aliases: []string{"n"},
+		Short:   "yaml n [--script/-s script_file] a.b.c newValueForC",
+		Example: `
+yaml new a.b.c cat
+yaml n a.b.c cat
+yaml n --script create_script.yaml
+      `,
+		Long: `Creates a new yaml w.r.t the given path and value.
+Outputs to STDOUT
+
+Create Scripts:
+Note that you can give a create script to perform more sophisticated yaml. This follows the same format as the update script.
+`,
+		Run: newProperty,
+	}
+	cmdNew.PersistentFlags().StringVarP(&writeScript, "script", "s", "", "yaml script for updating yaml")
+	return cmdNew
+}
+
 func readProperty(cmd *cobra.Command, args []string) {
+	if verbose {
+		backend.SetLevel(logging.DEBUG, "")
+	}
 	print(read(args))
 }
 
 func read(args []string) interface{} {
-	var parsedData map[interface{}]interface{}
+	var parsedData yaml.MapSlice
 
 	readData(args[0], &parsedData, inputJSON)
 
@@ -94,7 +131,39 @@ func read(args []string) interface{} {
 	return readMap(parsedData, paths[0], paths[1:len(paths)])
 }
 
+func newProperty(cmd *cobra.Command, args []string) {
+	if verbose {
+		backend.SetLevel(logging.DEBUG, "")
+	}
+	updatedData := newYaml(args)
+	print(updatedData)
+}
+
+func newYaml(args []string) interface{} {
+	var writeCommands map[string]interface{}
+	if writeScript != "" {
+		readData(writeScript, &writeCommands, false)
+	} else if len(args) < 2 {
+		die("Must provide <path_to_update> <value>")
+	} else {
+		writeCommands = make(map[string]interface{})
+		writeCommands[args[0]] = parseValue(args[1])
+	}
+
+	parsedData := make(yaml.MapSlice, 0)
+
+	for path, value := range writeCommands {
+		var paths = parsePath(path)
+		parsedData = writeMap(parsedData, paths, value)
+	}
+
+	return parsedData
+}
+
 func writeProperty(cmd *cobra.Command, args []string) {
+	if verbose {
+		backend.SetLevel(logging.DEBUG, "")
+	}
 	updatedData := updateYaml(args)
 	if writeInplace {
 		ioutil.WriteFile(args[0], []byte(yamlToString(updatedData)), 0644)
@@ -114,13 +183,14 @@ func updateYaml(args []string) interface{} {
 		writeCommands[args[1]] = parseValue(args[2])
 	}
 
-	var parsedData map[interface{}]interface{}
+	var parsedData yaml.MapSlice
 	readData(args[0], &parsedData, inputJSON)
 
 	for path, value := range writeCommands {
 		var paths = parsePath(path)
-		write(parsedData, paths[0], paths[1:len(paths)], value)
+		parsedData = writeMap(parsedData, paths, value)
 	}
+
 	return parsedData
 }
 

yaml_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"fmt"
 	"testing"
 )
 
@@ -26,11 +27,41 @@ func TestRead(t *testing.T) {
 	assertResult(t, 2, result)
 }
 
+func TestOrder(t *testing.T) {
+	result := read([]string{"order.yaml"})
+	formattedResult := yamlToString(result)
+	assertResult(t,
+		`version: 3
+application: MyApp`,
+		formattedResult)
+}
+
+func TestNewYaml(t *testing.T) {
+	result := newYaml([]string{"b.c", "3"})
+	formattedResult := fmt.Sprintf("%v", result)
+	assertResult(t,
+		"[{b [{c 3}]}]",
+		formattedResult)
+}
+
 func TestUpdateYaml(t *testing.T) {
-	updateYaml([]string{"sample.yaml", "b.c", "3"})
+	result := updateYaml([]string{"sample.yaml", "b.c", "3"})
+	formattedResult := fmt.Sprintf("%v", result)
+	assertResult(t,
+		"[{a Easy! as one two three} {b [{c 3} {d [3 4]} {e [[{name fred} {value 3}] [{name sam} {value 4}]]}]}]",
+		formattedResult)
 }
 
 func TestUpdateYaml_WithScript(t *testing.T) {
 	writeScript = "instruction_sample.yaml"
 	updateYaml([]string{"sample.yaml"})
 }
+
+func TestNewYaml_WithScript(t *testing.T) {
+	writeScript = "instruction_sample.yaml"
+	result := newYaml([]string{""})
+	formattedResult := fmt.Sprintf("%v", result)
+	assertResult(t,
+		"[{b [{c cat} {e [[{name Mike Farah}]]}]}]",
+		formattedResult)
+}