Version bump

README.md

@@ -12,6 +12,12 @@ On Ubuntu and other Linux distros supporting `snap` packages:
 ```
 snap install yq
 ```
+On Ubuntu 16.04 or higher from Debian package:
+```
+sudo add-apt-repository ppa:rmescandon/yq
+sudo apt update
+sudo apt install yq -y
+```
 or, [Download latest binary](https://github.com/mikefarah/yq/releases/latest) or alternatively:
 ```
 go get github.com/mikefarah/yq
@@ -77,4 +83,8 @@ Use "yq [command] --help" for more information about a command.
 2. add unit tests
 3. apply changes
 4. `make [local] build`
-5. profit
+5. If required, update the user documentation 
+    - Update README.md and/or documentation under the mkdocs folder
+    - `make [local] build-docs`
+    - browse to docs/index.html and check your changes 
+6. profit

commands_test.go

@@ -128,6 +128,15 @@ func TestReadCmd(t *testing.T) {
 	assertResult(t, "2\n", result.Output)
 }
 
+func TestReadMultiCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := runCmd(cmd, "read -d 1 examples/multiple_docs.yaml another.document")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	assertResult(t, "here\n", result.Output)
+}
+
 func TestReadCmd_ArrayYaml(t *testing.T) {
 	cmd := getRootCommand()
 	result := runCmd(cmd, "read examples/array.yaml [0].gather_facts")
@@ -363,6 +372,28 @@ func TestWriteCmd(t *testing.T) {
 	assertResult(t, expectedOutput, result.Output)
 }
 
+func TestWriteMultiCmd(t *testing.T) {
+	content := `b:
+  c: 3
+---
+apples: great
+`
+	filename := writeTempYamlFile(content)
+	defer removeTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := runCmd(cmd, fmt.Sprintf("write %s -d 1 apples ok", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `b:
+  c: 3
+---
+apples: ok
+`
+	assertResult(t, expectedOutput, result.Output)
+}
+
 func TestWriteCmd_EmptyArray(t *testing.T) {
 	content := `b: 3`
 	filename := writeTempYamlFile(content)
@@ -432,7 +463,7 @@ func TestWriteCmd_Inplace(t *testing.T) {
 	gotOutput := readTempYamlFile(filename)
 	expectedOutput := `b:
   c: 7`
-	assertResult(t, expectedOutput, gotOutput)
+	assertResult(t, expectedOutput, strings.Trim(gotOutput, "\n "))
 }
 
 func TestWriteCmd_Append(t *testing.T) {
@@ -472,6 +503,72 @@ b:
 	assertResult(t, expectedOutput, result.Output)
 }
 
+func TestDeleteYaml(t *testing.T) {
+	content := `a: 2
+b:
+  c: things
+  d: something else
+`
+	filename := writeTempYamlFile(content)
+	defer removeTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := runCmd(cmd, fmt.Sprintf("delete %s b.c", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `a: 2
+b:
+  d: something else
+`
+	assertResult(t, expectedOutput, result.Output)
+}
+
+func TestDeleteYamlArray(t *testing.T) {
+	content := `- 1
+- 2
+- 3	
+`
+	filename := writeTempYamlFile(content)
+	defer removeTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := runCmd(cmd, fmt.Sprintf("delete %s [1]", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `- 1
+- 3
+`
+	assertResult(t, expectedOutput, result.Output)
+}
+
+func TestDeleteYamlMulti(t *testing.T) {
+	content := `apples: great
+---
+- 1
+- 2
+- 3
+`
+	filename := writeTempYamlFile(content)
+	defer removeTempYamlFile(filename)
+
+	cmd := getRootCommand()
+	result := runCmd(cmd, fmt.Sprintf("delete -d 1 %s [1]", filename))
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+
+	expectedOutput := `apples: great
+---
+- 1
+- 3
+`
+	assertResult(t, expectedOutput, result.Output)
+}
+
 func TestMergeCmd(t *testing.T) {
 	cmd := getRootCommand()
 	result := runCmd(cmd, "merge examples/data1.yaml examples/data2.yaml")
@@ -488,6 +585,25 @@ c:
 	assertResult(t, expectedOutput, result.Output)
 }
 
+func TestMergeCmd_Multi(t *testing.T) {
+	cmd := getRootCommand()
+	result := runCmd(cmd, "merge -d1 examples/multiple_docs_small.yaml examples/data2.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `a: Easy! as one two three
+---
+a: other
+another:
+  document: here
+c:
+  test: 1
+---
+- 1
+- 2`
+	assertResult(t, expectedOutput, strings.Trim(result.Output, "\n "))
+}
+
 func TestMergeCmd_Error(t *testing.T) {
 	cmd := getRootCommand()
 	result := runCmd(cmd, "merge examples/data1.yaml")
@@ -504,7 +620,7 @@ func TestMergeCmd_ErrorUnreadableFile(t *testing.T) {
 	if result.Error == nil {
 		t.Error("Expected command to fail due to unknown file")
 	}
-	expectedOutput := `open fake-unknown: no such file or directory`
+	expectedOutput := `Error updating document at index 0: open fake-unknown: no such file or directory`
 	assertResult(t, expectedOutput, result.Error.Error())
 }
 
@@ -540,5 +656,5 @@ b:
 - 2
 c:
   test: 1`
-	assertResult(t, expectedOutput, gotOutput)
+	assertResult(t, expectedOutput, strings.Trim(gotOutput, "\n "))
 }

data_navigator.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"sort"
 	"strconv"
 
 	"gopkg.in/yaml.v2"
@@ -194,23 +193,6 @@ func calculateValue(value interface{}, tail []string) (interface{}, error) {
 	return value, nil
 }
 
-func mapToMapSlice(data map[interface{}]interface{}) yaml.MapSlice {
-	var mapSlice yaml.MapSlice
-
-	for k, v := range data {
-		if mv, ok := v.(map[interface{}]interface{}); ok {
-			v = mapToMapSlice(mv)
-		}
-		item := yaml.MapItem{Key: k, Value: v}
-		mapSlice = append(mapSlice, item)
-	}
-
-	// because the parsing of the yaml was done via a map the order will be inconsistent
-	// apply order to allow a consistent output
-	sort.SliceStable(mapSlice, func(i, j int) bool { return mapSlice[i].Key.(string) < mapSlice[j].Key.(string) })
-	return mapSlice
-}
-
 func deleteMap(context interface{}, paths []string) yaml.MapSlice {
 	log.Debugf("deleteMap for %v for %v\n", paths, context)
 

debian/changelog

@@ -0,0 +1,6 @@
+yq (1.15-0) bionic; urgency=medium
+
+  * Release 1.15
+
+ -- Roberto Mier Escandón <rmescandon@gmail.com>  Wed, 06 Jun 2018 11:32:03 +0200
+

debian/compat

@@ -0,0 +1 @@
+9

debian/control

@@ -0,0 +1,22 @@
+Source: yq
+Section: devel
+Priority: extra
+Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
+Build-Depends: debhelper (>= 9),
+               dh-golang,
+               golang-1.10-go,
+               rsync
+Standards-Version: 3.9.6
+Homepage: https://github.com/mikefarah/yq.git
+Vcs-Browser: https://github.com/mikefarah/yq.git
+Vcs-Git: https://github.com/mikefarah/yq.git
+
+Package: yq
+Architecture: all
+Built-Using: ${misc:Built-Using}
+Depends: ${shlibs:Depends},
+         ${misc:Depends}
+Description:
+ a lightweight and portable command-line YAML processor
+ .
+ The aim of the project is to be the [jq](https://github.com/stedolan/jq) or sed of yaml files.

debian/copyright

@@ -0,0 +1,7 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: yq
+Source: https://github.com/mikefarah/yq.git
+
+Files: *
+Copyright: 2017 Mike Farah Ltd. All rights reserved
+License: Proprietary

debian/gbp.conf

@@ -0,0 +1,2 @@
+[DEFAULT]
+pristine-tar = True

debian/rules

@@ -0,0 +1,59 @@
+#!/usr/bin/make -f
+#
+# Copyright (C) 2018 Roberto Mier Escandón <rmescandon@gmail.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+PROJECT := yq
+OWNER := mikefarah
+REPO := github.com
+GOVERSION := 1.10
+export DH_OPTIONS
+export DH_GOPKG := ${REPO}/${OWNER}/${PROJECT}
+export GOROOT := /usr/lib/go-${GOVERSION}
+export GOPATH := ${CURDIR}/_build
+export GOBIN := ${GOPATH}/bin
+export PATH := ${GOROOT}/bin:${GOBIN}:${PATH}
+BLDPATH := $(shell dpkg-architecture -qDEB_BUILD_GNU_TYPE)
+SRCDIR := ${CURDIR}/_build/src/${DH_GOPKG}
+DESTDIR := ${CURDIR}/debian/${PROJECT}
+BINDIR := /usr/bin
+ASSETSDIR := /usr/share/${PROJECT}
+
+%:
+	dh $@ --buildsystem=golang --with=golang
+
+override_dh_auto_build:
+	mkdir -p ${SRCDIR}
+	mkdir -p ${GOBIN}
+	# copy project to local srcdir to build from there
+	rsync -avz --progress --exclude=obj-${BLDPATH} --exclude=debian . $(SRCDIR)
+	# build go code
+	(cd ${SRCDIR} && go install ./...)
+
+override_dh_auto_test:
+	(cd ${SRCDIR} && go test -v ./...)
+
+override_dh_auto_install:
+	mkdir -p ${DESTDIR}/${BINDIR}
+	mkdir -p ${DESTDIR}/${ASSETSDIR}
+	cp ${CURDIR}/_build/bin/yq ${DESTDIR}/${BINDIR}
+	cp -rf ${SRCDIR}/LICENSE ${DESTDIR}/${ASSETSDIR}
+	cp -rf ${SRCDIR}/README.md ${DESTDIR}/${PLUGINSDIR}
+	chmod a+x ${DESTDIR}/${BINDIR}/yq
+
+override_dh_auto_clean:
+	dh_clean
+	rm -rf ${CURDIR}/obj-${BLDPATH}
+	rm -rf ${CURDIR}/_build

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

docs/index.html

@@ -373,6 +373,12 @@
 <pre><code>snap install yq
 </code></pre>
 
+<p>On Ubuntu 16.04 or higher from Debian package:</p>
+<pre><code>sudo add-apt-repository ppa:rmescandon/yq
+sudo apt update
+sudo apt install yq -y
+</code></pre>
+
 <p>or, <a href="https://github.com/mikefarah/yq/releases/latest">Download latest binary</a> or alternatively:</p>
 <pre><code>go get github.com/mikefarah/yq
 </code></pre>

docs/search/search_index.json

@@ -2,7 +2,7 @@
     "docs": [
         {
             "location": "/",
-            "text": "yq\n\u00b6\n\n\nyq is a lightweight and portable command-line YAML processor\n\n\nThe aim of the project is to be the \njq\n or sed of yaml files.\n\n\nInstall\n\u00b6\n\n\nOn MacOS:\n\n\nbrew install yq\n\n\n\n\nOn Ubuntu and other Linux distros supporting \nsnap\n packages:\n\n\nsnap install yq\n\n\n\n\nor, \nDownload latest binary\n or alternatively:\n\n\ngo get github.com/mikefarah/yq\n\n\n\n\nView on GitHub",
+            "text": "yq\n\u00b6\n\n\nyq is a lightweight and portable command-line YAML processor\n\n\nThe aim of the project is to be the \njq\n or sed of yaml files.\n\n\nInstall\n\u00b6\n\n\nOn MacOS:\n\n\nbrew install yq\n\n\n\n\nOn Ubuntu and other Linux distros supporting \nsnap\n packages:\n\n\nsnap install yq\n\n\n\n\nOn Ubuntu 16.04 or higher from Debian package:\n\n\nsudo add-apt-repository ppa:rmescandon/yq\nsudo apt update\nsudo apt install yq -y\n\n\n\n\nor, \nDownload latest binary\n or alternatively:\n\n\ngo get github.com/mikefarah/yq\n\n\n\n\nView on GitHub",
             "title": "Install"
         },
         {
@@ -12,7 +12,7 @@
         },
         {
             "location": "/#install",
-            "text": "On MacOS:  brew install yq  On Ubuntu and other Linux distros supporting  snap  packages:  snap install yq  or,  Download latest binary  or alternatively:  go get github.com/mikefarah/yq  View on GitHub",
+            "text": "On MacOS:  brew install yq  On Ubuntu and other Linux distros supporting  snap  packages:  snap install yq  On Ubuntu 16.04 or higher from Debian package:  sudo add-apt-repository ppa:rmescandon/yq\nsudo apt update\nsudo apt install yq -y  or,  Download latest binary  or alternatively:  go get github.com/mikefarah/yq  View on GitHub",
             "title": "Install"
         },
         {

docs/sitemap.xml

@@ -4,7 +4,7 @@
     
     <url>
      <loc>/</loc>
-     <lastmod>2018-05-07</lastmod>
+     <lastmod>2018-06-12</lastmod>
      <changefreq>daily</changefreq>
     </url>
     
@@ -12,7 +12,7 @@
     
     <url>
      <loc>/read/</loc>
-     <lastmod>2018-05-07</lastmod>
+     <lastmod>2018-06-12</lastmod>
      <changefreq>daily</changefreq>
     </url>
     
@@ -20,7 +20,7 @@
     
     <url>
      <loc>/write/</loc>
-     <lastmod>2018-05-07</lastmod>
+     <lastmod>2018-06-12</lastmod>
      <changefreq>daily</changefreq>
     </url>
     
@@ -28,7 +28,7 @@
     
     <url>
      <loc>/delete/</loc>
-     <lastmod>2018-05-07</lastmod>
+     <lastmod>2018-06-12</lastmod>
      <changefreq>daily</changefreq>
     </url>
     
@@ -36,7 +36,7 @@
     
     <url>
      <loc>/create/</loc>
-     <lastmod>2018-05-07</lastmod>
+     <lastmod>2018-06-12</lastmod>
      <changefreq>daily</changefreq>
     </url>
     
@@ -44,7 +44,7 @@
     
     <url>
      <loc>/convert/</loc>
-     <lastmod>2018-05-07</lastmod>
+     <lastmod>2018-06-12</lastmod>
      <changefreq>daily</changefreq>
     </url>
     
@@ -52,7 +52,7 @@
     
     <url>
      <loc>/merge/</loc>
-     <lastmod>2018-05-07</lastmod>
+     <lastmod>2018-06-12</lastmod>
      <changefreq>daily</changefreq>
     </url>
     

examples/multiple_docs.yaml

@@ -0,0 +1,15 @@
+a: Easy! as one two three
+b:
+  c: 2
+  d: [3, 4]
+  e:
+    - name: fred
+      value: 3
+    - name: sam
+      value: 4
+---
+another:
+  document: here
+---
+wow: 
+  - here is another

examples/multiple_docs_small.yaml

@@ -0,0 +1,7 @@
+a: Easy! as one two three
+---
+another:
+  document: here
+---
+- 1
+- 2

merge_test.go

@@ -1,30 +0,0 @@
-package main
-
-import (
-	"testing"
-
-	yaml "gopkg.in/yaml.v2"
-)
-
-func TestMerge(t *testing.T) {
-	result, _ := mergeYaml([]string{"examples/data1.yaml", "examples/data2.yaml", "examples/data3.yaml"})
-	expected := yaml.MapSlice{
-		yaml.MapItem{Key: "a", Value: "simple"},
-		yaml.MapItem{Key: "b", Value: []interface{}{1, 2}},
-		yaml.MapItem{Key: "c", Value: yaml.MapSlice{yaml.MapItem{Key: "other", Value: true}, yaml.MapItem{Key: "test", Value: 1}}},
-		yaml.MapItem{Key: "d", Value: false},
-	}
-	assertResultComplex(t, expected, result)
-}
-
-func TestMergeWithOverwrite(t *testing.T) {
-	overwriteFlag = true
-	result, _ := mergeYaml([]string{"examples/data1.yaml", "examples/data2.yaml", "examples/data3.yaml"})
-	expected := yaml.MapSlice{
-		yaml.MapItem{Key: "a", Value: "other"},
-		yaml.MapItem{Key: "b", Value: []interface{}{2, 3, 4}},
-		yaml.MapItem{Key: "c", Value: yaml.MapSlice{yaml.MapItem{Key: "other", Value: true}, yaml.MapItem{Key: "test", Value: 2}}},
-		yaml.MapItem{Key: "d", Value: false},
-	}
-	assertResultComplex(t, expected, result)
-}

mkdocs/index.md

@@ -12,6 +12,12 @@ On Ubuntu and other Linux distros supporting `snap` packages:
 ```
 snap install yq
 ```
+On Ubuntu 16.04 or higher from Debian package:
+```
+sudo add-apt-repository ppa:rmescandon/yq
+sudo apt update
+sudo apt install yq -y
+```
 or, [Download latest binary](https://github.com/mikefarah/yq/releases/latest) or alternatively:
 ```
 go get github.com/mikefarah/yq

release_instructions.txt

@@ -0,0 +1,30 @@
+- increment version in version.go
+- increment version in snapcraft.yaml
+- tag git with same version number
+- make sure local build passes
+- push tag to git
+- make local xcompile (builds binaries for all platforms)
+
+- git release
+  ./scripts/publish.sh
+
+- snapcraft
+  - will auto create a candidate, test it works then promote
+
+- brew
+  - create pull request pointing to latest git release
+
+- docker
+  - build and push latest and new version tag
+
+- debian package
+  - execute 
+    ```dch -i```
+  - fill debian/changelog with changes from last version
+  - build the package sources 
+    ```debuild -i -I -S -sa```
+    (signing with gpg key is required in order to put it to ppa)
+  - put to PPA
+    ```dput ppa:<REPOSITORY> ../yq_<VERSION>_source.changes```
+    (current distro repository is ppa:rmescandon/yq. In case that a new version
+    is released, please contact rmescandon@gmail.com to bump debian package)

scripts/publish.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-
+set -ex
 GITHUB_TOKEN="${GITHUB_TOKEN:?missing required input \'GITHUB_TOKEN\'}"
 
 CURRENT="$(git describe --tags --abbrev=0)"
@@ -8,32 +8,25 @@ OWNER="mikefarah"
 REPO="yq"
 
 release() {
-    mapfile -t logs < <(git log --pretty=oneline --abbrev-commit "${PREVIOUS}".."${CURRENT}")
-    description="$(printf '%s\n' "${logs[@]}")"
     github-release release \
         --user "$OWNER" \
         --repo "$REPO" \
-        --tag "$CURRENT" \
-        --description "$description" ||
-            github-release edit \
-                --user "$OWNER" \
-                --repo "$REPO" \
-                --tag "$CURRENT" \
-                --description "$description"
+        --tag "$CURRENT"
 }
 
 upload() {
-    mapfile -t files < <(find ./build -mindepth 1 -maxdepth 1)
-    for file in "${files[@]}"; do
+    while IFS=  read -r -d $'\0'; do
+        file=$REPLY
         BINARY=$(basename "${file}")
         echo "--> ${BINARY}"
         github-release upload \
+            --replace \
             --user "$OWNER" \
             --repo "$REPO" \
             --tag "$CURRENT" \
             --name "${BINARY}" \
             --file "$file"
-    done
+    done < <(find ./build -mindepth 1 -maxdepth 1 -print0)
 }
 
 release

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: 1.14.1
+version: 1.15.0
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.

vendor/vendor.json

@@ -20,6 +20,12 @@
 			"revision": "970db520ece77730c7e4724c61121037378659d9",
 			"revisionTime": "2016-03-15T20:05:05Z"
 		},
+		{
+			"checksumSHA1": "ljd3FhYRJ91cLZz3wsH9BQQ2JbA=",
+			"path": "github.com/pkg/errors",
+			"revision": "816c9085562cd7ee03e7f8188a1cfd942858cded",
+			"revisionTime": "2018-03-11T21:45:15Z"
+		},
 		{
 			"checksumSHA1": "xPKgXygsORkmXnLdtFaFmipYKaA=",
 			"path": "github.com/spf13/cobra",
@@ -33,10 +39,12 @@
 			"revisionTime": "2017-08-24T17:57:12Z"
 		},
 		{
-			"checksumSHA1": "RDJpJQwkF012L6m/2BJizyOksNw=",
+			"checksumSHA1": "DHNYKS5T54/XOqUsFFzdZMLEnVE=",
+			"origin": "github.com/mikefarah/yaml",
 			"path": "gopkg.in/yaml.v2",
-			"revision": "eb3733d160e74a9c7e442f435eb3bea458e1d19f",
-			"revisionTime": "2017-08-12T16:00:11Z"
+			"revision": "e175af14aaa1d0eff2ee04b691e4a4827a111416",
+			"revisionTime": "2018-06-13T04:05:11Z",
+			"tree": true
 		}
 	],
 	"rootPath": "github.com/mikefarah/yq"

version.go

@@ -11,12 +11,12 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "1.15.0"
+	Version = "2.0.0"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release
 	// such as "dev" (in development), "beta", "rc1", etc.
-	VersionPrerelease = ""
+	VersionPrerelease = "beta"
 )
 
 // ProductName is the name of the product

version_test.go

@@ -9,7 +9,7 @@ func TestGetVersionDisplay(t *testing.T) {
 	}{
 		{
 			name: "Display Version",
-			want: ProductName + " version " + Version + "\n",
+			want: ProductName + " version " + Version + "-" + VersionPrerelease + "\n",
 		},
 	}
 	for _, tt := range tests {

yq.go

@@ -1,13 +1,17 @@
 package main
 
 import (
-	"errors"
+	"bufio"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
+	"reflect"
 	"strconv"
 	"strings"
 
+	errors "github.com/pkg/errors"
+
 	logging "github.com/op/go-logging"
 	"github.com/spf13/cobra"
 	yaml "gopkg.in/yaml.v2"
@@ -20,17 +24,19 @@ var outputToJSON = false
 var overwriteFlag = false
 var verbose = false
 var version = false
+var docIndex = 0
 var log = logging.MustGetLogger("yq")
 
 func main() {
 	cmd := newCommandCLI()
 	if err := cmd.Execute(); err != nil {
-		fmt.Println(err.Error())
+		log.Error(err.Error())
 		os.Exit(1)
 	}
 }
 
 func newCommandCLI() *cobra.Command {
+	yaml.DefaultMapType = reflect.TypeOf(yaml.MapSlice{})
 	var rootCmd = &cobra.Command{
 		Use: "yq",
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -77,35 +83,37 @@ func newCommandCLI() *cobra.Command {
 }
 
 func createReadCmd() *cobra.Command {
-	return &cobra.Command{
+	var cmdRead = &cobra.Command{
 		Use:     "read [yaml_file] [path]",
 		Aliases: []string{"r"},
-		Short:   "yq r sample.yaml a.b.c",
+		Short:   "yq r [--doc/-d document_index] sample.yaml a.b.c",
 		Example: `
 yq read things.yaml a.b.c
 yq r - a.b.c (reads from stdin)
 yq r things.yaml a.*.c
-yq r things.yaml a.array[0].blah
+yq r -d1 things.yaml a.array[0].blah
 yq r things.yaml a.array[*].blah
       `,
 		Long: "Outputs the value of the given path in the yaml file to STDOUT",
 		RunE: readProperty,
 	}
+	cmdRead.PersistentFlags().IntVarP(&docIndex, "doc", "d", 0, "process document index number (0 based)")
+	return cmdRead
 }
 
 func createWriteCmd() *cobra.Command {
 	var cmdWrite = &cobra.Command{
 		Use:     "write [yaml_file] [path] [value]",
 		Aliases: []string{"w"},
-		Short:   "yq w [--inplace/-i] [--script/-s script_file] sample.yaml a.b.c newValueForC",
+		Short:   "yq w [--inplace/-i] [--script/-s script_file] [--doc/-d document_index] sample.yaml a.b.c newValueForC",
 		Example: `
 yq write things.yaml a.b.c cat
 yq write --inplace things.yaml a.b.c cat
 yq w -i things.yaml a.b.c cat
 yq w --script update_script.yaml things.yaml
 yq w -i -s update_script.yaml things.yaml
-yq w things.yaml a.b.d[+] foo
-yq w things.yaml a.b.d[+] foo
+yq w --doc 2 things.yaml a.b.d[+] foo
+yq w -d2 things.yaml a.b.d[+] foo
       `,
 		Long: `Updates the yaml file w.r.t the given path and value.
 Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead.
@@ -124,6 +132,7 @@ a.b.e:
 	}
 	cmdWrite.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace")
 	cmdWrite.PersistentFlags().StringVarP(&writeScript, "script", "s", "", "yaml script for updating yaml")
+	cmdWrite.PersistentFlags().IntVarP(&docIndex, "doc", "d", 0, "process document index number (0 based)")
 	return cmdWrite
 }
 
@@ -131,7 +140,7 @@ func createDeleteCmd() *cobra.Command {
 	var cmdDelete = &cobra.Command{
 		Use:     "delete [yaml_file] [path]",
 		Aliases: []string{"d"},
-		Short:   "yq d [--inplace/-i] sample.yaml a.b.c",
+		Short:   "yq d [--inplace/-i] [--doc/-d document_index] sample.yaml a.b.c",
 		Example: `
 yq delete things.yaml a.b.c
 yq delete --inplace things.yaml a.b.c
@@ -144,6 +153,7 @@ Outputs to STDOUT unless the inplace flag is used, in which case the file is upd
 		RunE: deleteProperty,
 	}
 	cmdDelete.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace")
+	cmdDelete.PersistentFlags().IntVarP(&docIndex, "doc", "d", 0, "process document index number (0 based)")
 	return cmdDelete
 }
 
@@ -173,7 +183,7 @@ func createMergeCmd() *cobra.Command {
 	var cmdMerge = &cobra.Command{
 		Use:     "merge [initial_yaml_file] [additional_yaml_file]...",
 		Aliases: []string{"m"},
-		Short:   "yq m [--inplace/-i] [--overwrite/-x] sample.yaml sample2.yaml",
+		Short:   "yq m [--inplace/-i] [--doc/-d document_index] [--overwrite/-x] sample.yaml sample2.yaml",
 		Example: `
 yq merge things.yaml other.yaml
 yq merge --inplace things.yaml other.yaml
@@ -190,6 +200,7 @@ If overwrite flag is set then existing values will be overwritten using the valu
 	}
 	cmdMerge.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace")
 	cmdMerge.PersistentFlags().BoolVarP(&overwriteFlag, "overwrite", "x", false, "update the yaml file by overwriting existing values")
+	cmdMerge.PersistentFlags().IntVarP(&docIndex, "doc", "d", 0, "process document index number (0 based)")
 	return cmdMerge
 }
 
@@ -207,7 +218,6 @@ func readProperty(cmd *cobra.Command, args []string) error {
 }
 
 func read(args []string) (interface{}, error) {
-	var parsedData yaml.MapSlice
 	var path = ""
 
 	if len(args) < 1 {
@@ -215,67 +225,17 @@ func read(args []string) (interface{}, error) {
 	} else if len(args) > 1 {
 		path = args[1]
 	}
-
-	if err := readData(args[0], &parsedData); err != nil {
 	var generalData interface{}
-		if err = readData(args[0], &generalData); err != nil {
+	if err := readData(args[0], docIndex, &generalData); err != nil {
 		return nil, err
 	}
-		item := yaml.MapItem{Key: "thing", Value: generalData}
-		parsedData = yaml.MapSlice{item}
-		path = "thing." + path
-	}
-
-	if parsedData != nil && parsedData[0].Key == nil {
-		var interfaceData []map[interface{}]interface{}
-		if err := readData(args[0], &interfaceData); err == nil {
-			var listMap []yaml.MapSlice
-			for _, item := range interfaceData {
-				listMap = append(listMap, mapToMapSlice(item))
-			}
-			return readYamlArray(listMap, path)
-		}
-	}
-
 	if path == "" {
-		return parsedData, nil
+		return generalData, nil
 	}
 
 	var paths = parsePath(path)
-
-	return readMap(parsedData, paths[0], paths[1:])
-}
-
-func readYamlArray(listMap []yaml.MapSlice, path string) (interface{}, error) {
-	if path == "" {
-		return listMap, nil
-	}
-
-	var paths = parsePath(path)
-
-	if paths[0] == "*" {
-		if len(paths[1:]) == 0 {
-			return listMap, nil
-		}
-		var results []interface{}
-		for _, m := range listMap {
-			value, err := readMap(m, paths[1], paths[2:])
-			if err != nil {
-				return nil, err
-			}
-			results = append(results, value)
-		}
-		return results, nil
-	}
-
-	index, err := strconv.ParseInt(paths[0], 10, 64)
-	if err != nil {
-		return nil, fmt.Errorf("Error accessing array: %v", err)
-	}
-	if len(paths[1:]) == 0 {
-		return listMap[index], nil
-	}
-	return readMap(listMap[index], paths[1], paths[2:])
+	value, err := recurse(generalData, paths[0], paths[1:])
+	return value, err
 }
 
 func newProperty(cmd *cobra.Command, args []string) error {
@@ -292,159 +252,175 @@ func newProperty(cmd *cobra.Command, args []string) error {
 }
 
 func newYaml(args []string) (interface{}, error) {
-	var writeCommands yaml.MapSlice
-	if writeScript != "" {
-		if err := readData(writeScript, &writeCommands); err != nil {
-			return nil, err
-		}
-	} else if len(args) < 2 {
-		return nil, errors.New("Must provide <path_to_update> <value>")
-	} else {
-		writeCommands = make(yaml.MapSlice, 1)
-		writeCommands[0] = yaml.MapItem{Key: args[0], Value: parseValue(args[1])}
+	var writeCommands, writeCommandsError = readWriteCommands(args, 2, "Must provide <path_to_update> <value>")
+	if writeCommandsError != nil {
+		return nil, writeCommandsError
 	}
 
-	var parsedData yaml.MapSlice
-	var prependCommand = ""
+	var dataBucket interface{}
 	var isArray = strings.HasPrefix(writeCommands[0].Key.(string), "[")
 	if isArray {
-		item := yaml.MapItem{Key: "thing", Value: make(yaml.MapSlice, 0)}
-		parsedData = yaml.MapSlice{item}
-		prependCommand = "thing"
+		dataBucket = make([]interface{}, 0)
 	} else {
-		parsedData = make(yaml.MapSlice, 0)
+		dataBucket = make(yaml.MapSlice, 0)
 	}
 
-	return updateParsedData(parsedData, writeCommands, prependCommand)
+	for _, entry := range writeCommands {
+		path := entry.Key.(string)
+		value := entry.Value
+		log.Debugf("setting %v to %v", path, value)
+		var paths = parsePath(path)
+		dataBucket = updatedChildValue(dataBucket, paths, value)
+	}
+
+	return dataBucket, nil
+}
+
+type updateDataFn func(dataBucket interface{}, currentIndex int) (interface{}, error)
+
+func mapYamlDecoder(updateData updateDataFn, encoder *yaml.Encoder) yamlDecoderFn {
+	return func(decoder *yaml.Decoder) error {
+		var dataBucket interface{}
+		var errorReading error
+		var errorWriting error
+		var errorUpdating error
+		var currentIndex = 0
+
+		for {
+			log.Debugf("Read doc %v", currentIndex)
+			errorReading = decoder.Decode(&dataBucket)
+
+			if errorReading == io.EOF {
+				if currentIndex < docIndex {
+					return fmt.Errorf("Asked to process document %v but there are only %v document(s)", docIndex, currentIndex)
+				}
+				return nil
+			} else if errorReading != nil {
+				return errors.Wrapf(errorReading, "Error reading document at index %v, %v", currentIndex, errorReading)
+			}
+			dataBucket, errorUpdating = updateData(dataBucket, currentIndex)
+			if errorUpdating != nil {
+				return errors.Wrapf(errorUpdating, "Error updating document at index %v", currentIndex)
+			}
+
+			errorWriting = encoder.Encode(dataBucket)
+
+			if errorWriting != nil {
+				return errors.Wrapf(errorWriting, "Error writing document at index %v, %v", currentIndex, errorWriting)
+			}
+			currentIndex = currentIndex + 1
+		}
+	}
 }
 
 func writeProperty(cmd *cobra.Command, args []string) error {
-	updatedData, err := updateYaml(args)
-	if err != nil {
-		return err
+	var writeCommands, writeCommandsError = readWriteCommands(args, 3, "Must provide <filename> <path_to_update> <value>")
+	if writeCommandsError != nil {
+		return writeCommandsError
 	}
-	return write(cmd, args[0], updatedData)
+	var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) {
+		if currentIndex == docIndex {
+			log.Debugf("Updating doc %v", currentIndex)
+			for _, entry := range writeCommands {
+				path := entry.Key.(string)
+				value := entry.Value
+				log.Debugf("setting %v to %v", path, value)
+				var paths = parsePath(path)
+				dataBucket = updatedChildValue(dataBucket, paths, value)
+			}
+		}
+		return dataBucket, nil
+	}
+	return readAndUpdate(cmd.OutOrStdout(), args[0], updateData)
 }
 
-func write(cmd *cobra.Command, filename string, updatedData interface{}) error {
+func readAndUpdate(stdOut io.Writer, inputFile string, updateData updateDataFn) error {
+	var destination io.Writer
+	var destinationName string
 	if writeInplace {
-		dataStr, err := yamlToString(updatedData)
+		var tempFile, err = ioutil.TempFile("", "temp")
 		if err != nil {
 			return err
 		}
-		return ioutil.WriteFile(filename, []byte(dataStr), 0644)
+		destinationName = tempFile.Name()
+		destination = tempFile
+		defer func() {
+			safelyCloseFile(tempFile)
+			safelyRenameFile(tempFile.Name(), inputFile)
+		}()
+	} else {
+		var writer = bufio.NewWriter(stdOut)
+		destination = writer
+		destinationName = "Stdout"
+		defer safelyFlush(writer)
 	}
-	dataStr, err := toString(updatedData)
-	if err != nil {
-		return err
-	}
-	cmd.Println(dataStr)
-	return nil
+	var encoder = yaml.NewEncoder(destination)
+	log.Debugf("Writing to %v from %v", destinationName, inputFile)
+	return readStream(inputFile, mapYamlDecoder(updateData, encoder))
 }
 
 func deleteProperty(cmd *cobra.Command, args []string) error {
-	updatedData, err := deleteYaml(args)
-	if err != nil {
-		return err
-	}
-	return write(cmd, args[0], updatedData)
-}
-
-func deleteYaml(args []string) (interface{}, error) {
-	var parsedData yaml.MapSlice
-	var deletePath string
-
 	if len(args) < 2 {
-		return nil, errors.New("Must provide <filename> <path_to_delete>")
+		return errors.New("Must provide <filename> <path_to_delete>")
+	}
+	var deletePath = args[1]
+	var paths = parsePath(deletePath)
+	var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) {
+		if currentIndex == docIndex {
+			log.Debugf("Deleting path in doc %v", currentIndex)
+			return deleteChildValue(dataBucket, paths), nil
+		}
+		return dataBucket, nil
 	}
 
-	deletePath = args[1]
-
-	if err := readData(args[0], &parsedData); err != nil {
-		var generalData interface{}
-		if err = readData(args[0], &generalData); err != nil {
-			return nil, err
-		}
-		item := yaml.MapItem{Key: "thing", Value: generalData}
-		parsedData = yaml.MapSlice{item}
-		deletePath = "thing." + deletePath
-	}
-
-	path := parsePath(deletePath)
-	return deleteMap(parsedData, path), nil
+	return readAndUpdate(cmd.OutOrStdout(), args[0], updateData)
 }
 
 func mergeProperties(cmd *cobra.Command, args []string) error {
 	if len(args) < 2 {
 		return errors.New("Must provide at least 2 yaml files")
 	}
+	var input = args[0]
+	var filesToMerge = args[1:]
 
-	updatedData, err := mergeYaml(args)
-	if err != nil {
-		return err
-	}
-	return write(cmd, args[0], updatedData)
-}
-
-func mergeYaml(args []string) (interface{}, error) {
-	var updatedData map[interface{}]interface{}
-
-	for _, f := range args {
-		var parsedData map[interface{}]interface{}
-		if err := readData(f, &parsedData); err != nil {
+	var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) {
+		if currentIndex == docIndex {
+			log.Debugf("Merging doc %v", currentIndex)
+			var mergedData map[interface{}]interface{}
+			if err := merge(&mergedData, dataBucket, overwriteFlag); err != nil {
 				return nil, err
 			}
-		if err := merge(&updatedData, parsedData, overwriteFlag); err != nil {
+			for _, f := range filesToMerge {
+				var fileToMerge interface{}
+				if err := readData(f, 0, &fileToMerge); err != nil {
+					return nil, err
+				}
+				if err := merge(&mergedData, fileToMerge, overwriteFlag); err != nil {
 					return nil, err
 				}
 			}
-
-	return mapToMapSlice(updatedData), nil
+			return mergedData, nil
+		}
+		return dataBucket, nil
+	}
+	yaml.DefaultMapType = reflect.TypeOf(map[interface{}]interface{}{})
+	defer func() { yaml.DefaultMapType = reflect.TypeOf(yaml.MapSlice{}) }()
+	return readAndUpdate(cmd.OutOrStdout(), input, updateData)
 }
 
-func updateParsedData(parsedData yaml.MapSlice, writeCommands yaml.MapSlice, prependCommand string) (interface{}, error) {
-	var prefix = ""
-	if prependCommand != "" {
-		prefix = prependCommand + "."
-	}
-	for _, entry := range writeCommands {
-		path := prefix + entry.Key.(string)
-		value := entry.Value
-		var paths = parsePath(path)
-		parsedData = writeMap(parsedData, paths, value)
-	}
-	if prependCommand != "" {
-		return readMap(parsedData, prependCommand, make([]string, 0))
-	}
-	return parsedData, nil
-}
-
-func updateYaml(args []string) (interface{}, error) {
+func readWriteCommands(args []string, expectedArgs int, badArgsMessage string) (yaml.MapSlice, error) {
 	var writeCommands yaml.MapSlice
-	var prependCommand = ""
 	if writeScript != "" {
-		if err := readData(writeScript, &writeCommands); err != nil {
+		if err := readData(writeScript, 0, &writeCommands); err != nil {
 			return nil, err
 		}
-	} else if len(args) < 3 {
-		return nil, errors.New("Must provide <filename> <path_to_update> <value>")
+	} else if len(args) < expectedArgs {
+		return nil, errors.New(badArgsMessage)
 	} else {
 		writeCommands = make(yaml.MapSlice, 1)
-		writeCommands[0] = yaml.MapItem{Key: args[1], Value: parseValue(args[2])}
+		writeCommands[0] = yaml.MapItem{Key: args[expectedArgs-2], Value: parseValue(args[expectedArgs-1])}
 	}
-
-	var parsedData yaml.MapSlice
-	if err := readData(args[0], &parsedData); err != nil {
-		var generalData interface{}
-		if err = readData(args[0], &generalData); err != nil {
-			return nil, err
-		}
-		item := yaml.MapItem{Key: "thing", Value: generalData}
-		parsedData = yaml.MapSlice{item}
-		prependCommand = "thing"
-	}
-
-	return updateParsedData(parsedData, writeCommands, prependCommand)
+	return writeCommands, nil
 }
 
 func parseValue(argument string) interface{} {
@@ -487,7 +463,7 @@ func marshalContext(context interface{}) (string, error) {
 	out, err := yaml.Marshal(context)
 
 	if err != nil {
-		return "", fmt.Errorf("error printing yaml: %v", err)
+		return "", errors.Wrap(err, "error printing yaml")
 	}
 
 	outStr := string(out)
@@ -499,22 +475,59 @@ func marshalContext(context interface{}) (string, error) {
 	return outStr, nil
 }
 
-func readData(filename string, parsedData interface{}) error {
+func safelyRenameFile(from string, to string) {
+	if err := os.Rename(from, to); err != nil {
+		log.Errorf("Error renaming from %v to %v", from, to)
+		log.Error(err.Error())
+	}
+}
+
+func safelyFlush(writer *bufio.Writer) {
+	if err := writer.Flush(); err != nil {
+		log.Error("Error flushing writer!")
+		log.Error(err.Error())
+	}
+
+}
+func safelyCloseFile(file *os.File) {
+	err := file.Close()
+	if err != nil {
+		log.Error("Error closing file!")
+		log.Error(err.Error())
+	}
+}
+
+type yamlDecoderFn func(*yaml.Decoder) error
+
+func readStream(filename string, yamlDecoder yamlDecoderFn) error {
 	if filename == "" {
 		return errors.New("Must provide filename")
 	}
 
-	var rawData []byte
-	var err error
+	var stream io.Reader
 	if filename == "-" {
-		rawData, err = ioutil.ReadAll(os.Stdin)
+		stream = bufio.NewReader(os.Stdin)
 	} else {
-		rawData, err = ioutil.ReadFile(filename)
-	}
-
+		file, err := os.Open(filename)
 		if err != nil {
 			return err
 		}
-
-	return yaml.Unmarshal(rawData, parsedData)
+		defer safelyCloseFile(file)
+		stream = file
+	}
+	return yamlDecoder(yaml.NewDecoder(stream))
+}
+
+func readData(filename string, indexToRead int, parsedData interface{}) error {
+	return readStream(filename, func(decoder *yaml.Decoder) error {
+		// naive implementation of document indexing, decodes all the yaml documents
+		// before the docIndex and throws them away.
+		for currentIndex := 0; currentIndex < indexToRead; currentIndex++ {
+			errorSkipping := decoder.Decode(parsedData)
+			if errorSkipping != nil {
+				return errors.Wrapf(errorSkipping, "Error processing document at index %v, %v", currentIndex, errorSkipping)
+			}
+		}
+		return decoder.Decode(parsedData)
+	})
 }

yq_test.go

@@ -28,6 +28,13 @@ func TestRead(t *testing.T) {
 	assertResult(t, 2, result)
 }
 
+func TestReadMulti(t *testing.T) {
+	docIndex = 1
+	result, _ := read([]string{"examples/multiple_docs.yaml", "another.document"})
+	assertResult(t, "here", result)
+	docIndex = 0
+}
+
 func TestReadArray(t *testing.T) {
 	result, _ := read([]string{"examples/sample_array.yaml", "[1]"})
 	assertResult(t, 2, result)
@@ -71,37 +78,6 @@ func TestNewYamlArray(t *testing.T) {
 		formattedResult)
 }
 
-func TestUpdateYaml(t *testing.T) {
-	result, _ := updateYaml([]string{"examples/sample.yaml", "b.c", "3"})
-	formattedResult := fmt.Sprintf("%v", result)
-	assertResult(t,
-		"[{a Easy! as one two three} {b [{c 3} {d [3 4]} {e [[{name fred} {value 3}] [{name sam} {value 4}]]}]}]",
-		formattedResult)
-}
-
-func TestUpdateYamlArray(t *testing.T) {
-	result, _ := updateYaml([]string{"examples/sample_array.yaml", "[0]", "3"})
-	formattedResult := fmt.Sprintf("%v", result)
-	assertResult(t,
-		"[3 2 3]",
-		formattedResult)
-}
-
-func TestUpdateYaml_WithScript(t *testing.T) {
-	writeScript = "examples/instruction_sample.yaml"
-	_, _ = updateYaml([]string{"examples/sample.yaml"})
-}
-
-func TestUpdateYaml_WithUnknownScript(t *testing.T) {
-	writeScript = "fake-unknown"
-	_, err := updateYaml([]string{"examples/sample.yaml"})
-	if err == nil {
-		t.Error("Expected error due to unknown file")
-	}
-	expectedOutput := `open fake-unknown: no such file or directory`
-	assertResult(t, expectedOutput, err.Error())
-}
-
 func TestNewYaml_WithScript(t *testing.T) {
 	writeScript = "examples/instruction_sample.yaml"
 	expectedResult := `b:
@@ -122,11 +98,3 @@ func TestNewYaml_WithUnknownScript(t *testing.T) {
 	expectedOutput := `open fake-unknown: no such file or directory`
 	assertResult(t, expectedOutput, err.Error())
 }
-
-func TestDeleteYaml(t *testing.T) {
-	result, _ := deleteYaml([]string{"examples/sample.yaml", "b.c"})
-	formattedResult := fmt.Sprintf("%v", result)
-	assertResult(t,
-		"[{a Easy! as one two three} {b [{d [3 4]} {e [[{name fred} {value 3}] [{name sam} {value 4}]]}]}]",
-		formattedResult)
-}