More scenarios

alias, anchor and explode ops are now all documented together

.github/ISSUE_TEMPLATE/bug_report_v3.md

@@ -1,8 +1,8 @@
 ---
-name: Bug report
+name: Bug report - V3
 about: Create a report to help us improve
 title: ''
-labels: bug
+labels: bug, v3
 assignees: ''
 
 ---
@@ -10,11 +10,12 @@ assignees: ''
 **Describe the bug**
 A clear and concise description of what the bug is.
 
-version of yq: 
+Version of yq: 3.X.X
-operating system:
+Operating system: mac/linux/windows/....
+Installed via: docker/binary release/homebrew/snap/...
 
 **Input Yaml**
-Concise yaml document(s) (as simple as possible to show the bug)
+Concise yaml document(s) (as simple as possible to show the bug, please keep it to 10 lines or less)
 data1.yml:
 ```yaml
 this: should really work

.github/ISSUE_TEMPLATE/bug_report_v4.md

@@ -0,0 +1,49 @@
+---
+name: Bug report - V4
+about: Create a report to help us improve
+title: ''
+labels: bug, v4
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+Version of yq: 4.X.X
+Operating system: mac/linux/windows/....
+Installed via: docker/binary release/homebrew/snap/...
+
+**Input Yaml**
+Concise yaml document(s) (as simple as possible to show the bug, please keep it to 10 lines or less)
+data1.yml:
+```yaml
+this: should really work
+```
+
+data2.yml:
+```yaml
+but: it strangely didn't
+```
+
+**Command**
+The command you ran:
+```
+yq eval-all 'select(fileIndex==0) | .a.b.c' data1.yml data2.yml
+```
+
+**Actual behavior**
+
+```yaml
+cat: meow
+```
+
+**Expected behavior**
+
+```yaml
+this: should really work
+but: it strangely didn't
+```
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,17 +1,21 @@
 ---
-name: Feature request
+name: Feature request - V4
 about: Suggest an idea for this project
 title: ''
-labels: enhancement
+labels: enhancement, v4
 assignees: ''
 
 ---
 
-**Is your feature request related to a problem? Please describe.**
+**Please describe your feature request.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+A clear and concise description of what the request is and what it would solve. 
+Ex. I wish I could use yq to [...]
+
+Please note that V3 will no longer have any enhancements.
 
 **Describe the solution you'd like**
 If we have data1.yml like:
+(please keep to around 10 lines )
 
 ```yaml
 country: Australia
@@ -20,7 +24,7 @@ country: Australia
 And we run a command:
 
 ```bash
-yq predictWeather data1.yml
+yq eval 'predictWeatherOf(.country)'
 ```
 
 it could output

.github/workflows/go.yml

@@ -28,7 +28,4 @@ jobs:
       run: |
         export PATH=${PATH}:`go env GOPATH`/bin
         scripts/devtools.sh
-    - name: Build
-      run: |
-        export PATH=${PATH}:`go env GOPATH`/bin
         make local build

.github/workflows/release.yml

@@ -0,0 +1,78 @@
+name: Release YQ
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  publishGitRelease:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.15'
+      - name: Cross compile
+        run: |
+          sudo apt-get install rhash -y
+          go get github.com/mitchellh/gox
+          ./scripts/xcompile.sh
+
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1.0.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: ${{ github.ref }}
+          draft: true
+          prerelease: false
+
+      - uses: shogo82148/actions-upload-release-asset@v1
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: build/*
+
+  publishDocker:
+    environment: dockerhub
+    env:
+      IMAGE_NAME: mikefarah/yq
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get latest release tag      
+        uses: oprypin/find-latest-tag@v1
+        with:
+          repository: mikefarah/yq  # The repository to scan.
+          releases-only: true  # We know that all relevant tags have a GitHub release for them.
+        id: yq  
+        
+      - name: Clone source code
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ steps.yq.outputs.tag }}
+      
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: all
+  
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          version: latest
+
+      - name: Available platforms
+        run: echo ${{ steps.buildx.outputs.platforms }} && docker version
+ 
+      - name: Build and push image
+        run: |
+          IMAGE_VERSION="$(git describe --tags --abbrev=0)"
+          SHORT_SHA1=$(git rev-parse --short HEAD)
+          PLATFORMS="linux/amd64,linux/ppc64le,linux/arm64"
+          echo "Building and pushing version ${IMAGE_VERSION} of image ${IMAGE_NAME}"
+          echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+          docker buildx build --platform "${PLATFORMS}" -t "${IMAGE_NAME}:${IMAGE_VERSION}"  -t "${IMAGE_NAME}:latest" -t "${IMAGE_NAME}:4" \
+            --push .
+    

Dockerfile

@@ -12,7 +12,7 @@ RUN CGO_ENABLED=0 make local build
 
 # Choose alpine as a base image to make this useful for CI, as many
 # CI tools expect an interactive shell inside the container
-FROM alpine:3.12 as production
+FROM alpine:3.12.3 as production
 
 COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq
 RUN chmod +x /usr/bin/yq

README.md

@@ -7,11 +7,17 @@ a lightweight and portable command-line YAML processor
 
 The aim of the project is to be the [jq](https://github.com/stedolan/jq) or sed of yaml files.
 
+## V4 released!
+V4 is now officially released, it's quite different from V3 (sorry for the migration), however it is much more similar to ```jq```, using a similar expression syntax and therefore support much more complex functionality! 
+
+If you've been using v3 and want/need to upgrade, checkout the [upgrade guide](https://mikefarah.gitbook.io/yq/v/v4.x/upgrading-from-v3).
+
 ## Install
 
 ### [Download the latest binary](https://github.com/mikefarah/yq/releases/latest)
 
 ### MacOS:
+Using [Homebrew](https://brew.sh/)
 ```
 brew install yq
 ```
@@ -44,13 +50,12 @@ rm /etc/myfile.tmp
 Use wget to download the pre-compiled binaries:
 
 ```bash
-wget https://github.com/mikefarah/yq/releases/download/{VERSION}/{BINARY} -O /usr/bin/yq &&\
+wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/bin/yq &&\
     chmod +x /usr/bin/yq
 ```
 
 For instance, VERSION=4.0.0 and BINARY=yq_linux_amd64
 
-
 ### Run with Docker
 
 #### Oneshot use:
@@ -75,7 +80,7 @@ yq() {
 
 ### Go Get:
 ```
-GO111MODULE=on go get github.com/mikefarah/yq/v4
+GO111MODULE=on go get github.com/mikefarah/yq
 ```
 
 ## Community Supported Installation methods
@@ -88,6 +93,14 @@ choco install yq
 ```
 Supported by @chillum (https://chocolatey.org/packages/yq)
 
+### Mac:
+Using [MacPorts](https://www.macports.org/)
+```
+sudo port selfupdate
+sudo port install yq
+```
+Supported by @herbygillot (https://ports.macports.org/maintainer/github/herbygillot)
+
 ### Alpine Linux
 - Enable edge/community repo by adding ```$MIRROR/alpine/edge/community``` to ```/etc/apk/repositories```
 - Update database index with ```apk update```
@@ -113,13 +126,13 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
 - Colorized yaml output
 - [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/v/v4.x/traverse)
 - [Sort yaml by keys](https://mikefarah.gitbook.io/yq/v/v4.x/sort-keys)
-- Manipulate yaml [comments](https://app.gitbook.com/@mikefarah/s/yq/v/v4.x/comment-operators), [styling](https://app.gitbook.com/@mikefarah/s/yq/v/v4.x/style), [tags](https://app.gitbook.com/@mikefarah/s/yq/v/v4.x/tag) and [anchors](https://app.gitbook.com/@mikefarah/s/yq/v/v4.x/explode).
+- Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/comment-operators), [styling](https://mikefarah.gitbook.io/yq/style), [tags](https://mikefarah.gitbook.io/yq/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/anchor-and-alias-operators).
 - [Update yaml inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
 - [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/v/v4.x/select#select-and-update-matching-values-in-map)
 - Keeps yaml formatting and comments when updating (though there are issues with whitespace)
 - [Convert to/from json to yaml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
 - [Pipe data in by using '-'](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate)
-- General shell completion scripts (bash/zsh/fish/powershell) (https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
+- [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
 
 ## [Usage](https://mikefarah.gitbook.io/yq/)
 
@@ -158,8 +171,5 @@ Simple Example:
 yq e '.a.b | length' f1.yml f2.yml 
 ```
 
-## Upgrade from V3
-If you've been using v3 and want/need to upgrade, checkout the [upgrade guide](https://mikefarah.gitbook.io/yq/v/v4.x/upgrading-from-v3).
-
 ## Known Issues / Missing Features
 - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)

cmd/root.go

@@ -47,6 +47,7 @@ func New() *cobra.Command {
 	rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output")
 	rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
 	rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace of first yaml file given.")
+	rootCmd.PersistentFlags().BoolVarP(&unwrapScalar, "unwrapScalar", "", true, "unwrap scalar, print the value with no quotes, colors or comments")
 	rootCmd.PersistentFlags().BoolVarP(&exitStatus, "exit-status", "e", false, "set exit status if there are no matches or null or false is returned")
 
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.0.0-beta1"
+	Version = "4.1.0"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

go.sum

@@ -263,6 +263,7 @@ golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

pkg/yqlib/candidate_node.go

@@ -21,6 +21,14 @@ func (n *CandidateNode) GetKey() string {
 	return fmt.Sprintf("%v - %v", n.Document, n.Path)
 }
 
+func (n *CandidateNode) CreateChildPath(path interface{}) []interface{} {
+	//don't use append as they may actually modify the path of the orignal node!
+	newPath := make([]interface{}, len(n.Path)+1)
+	copy(newPath, n.Path)
+	newPath[len(n.Path)] = path
+	return newPath
+}
+
 func (n *CandidateNode) Copy() (*CandidateNode, error) {
 	clone := &CandidateNode{}
 	err := copier.Copy(clone, n)

pkg/yqlib/doc/Alternative (Default value).md

@@ -0,0 +1,73 @@
+This operator is used to provide alternative (or default) values when a particular expression is either null or false.
+
+## LHS is defined
+Given a sample.yml file of:
+```yaml
+a: bridge
+```
+then
+```bash
+yq eval '.a // "hello"' sample.yml
+```
+will output
+```yaml
+bridge
+```
+
+## LHS is not defined
+Given a sample.yml file of:
+```yaml
+{}
+```
+then
+```bash
+yq eval '.a // "hello"' sample.yml
+```
+will output
+```yaml
+hello
+```
+
+## LHS is null
+Given a sample.yml file of:
+```yaml
+a: ~
+```
+then
+```bash
+yq eval '.a // "hello"' sample.yml
+```
+will output
+```yaml
+hello
+```
+
+## LHS is false
+Given a sample.yml file of:
+```yaml
+a: false
+```
+then
+```bash
+yq eval '.a // "hello"' sample.yml
+```
+will output
+```yaml
+hello
+```
+
+## RHS is an expression
+Given a sample.yml file of:
+```yaml
+a: false
+b: cat
+```
+then
+```bash
+yq eval '.a // .b' sample.yml
+```
+will output
+```yaml
+cat
+```
+

pkg/yqlib/doc/Anchor and Alias Operators.md

@@ -1,4 +1,67 @@
-Explodes (or dereferences) aliases and anchors.
+Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference aliases and remove anchor names).
+
+`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
+
+
+## Get anchor
+Given a sample.yml file of:
+```yaml
+a: &billyBob cat
+```
+then
+```bash
+yq eval '.a | anchor' sample.yml
+```
+will output
+```yaml
+billyBob
+```
+
+## Set anchor
+Given a sample.yml file of:
+```yaml
+a: cat
+```
+then
+```bash
+yq eval '.a anchor = "foobar"' sample.yml
+```
+will output
+```yaml
+a: &foobar cat
+```
+
+## Get alias
+Given a sample.yml file of:
+```yaml
+b: &billyBob meow
+a: *billyBob
+```
+then
+```bash
+yq eval '.a | alias' sample.yml
+```
+will output
+```yaml
+billyBob
+```
+
+## Set alias
+Given a sample.yml file of:
+```yaml
+b: &meow purr
+a: cat
+```
+then
+```bash
+yq eval '.a alias = "meow"' sample.yml
+```
+will output
+```yaml
+b: &meow purr
+a: *meow
+```
+
 ## Explode alias and anchor
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Assign.md

@@ -112,7 +112,7 @@ a:
 ```
 then
 ```bash
-yq eval '(.a.[] | select(. == "apple")) = "frog"' sample.yml
+yq eval '(.a[] | select(. == "apple")) = "frog"' sample.yml
 ```
 will output
 ```yaml

pkg/yqlib/doc/Delete.md

@@ -14,6 +14,23 @@ will output
 a: cat
 ```
 
+## Delete nested entry in map
+Given a sample.yml file of:
+```yaml
+a:
+  a1: fred
+  a2: frood
+```
+then
+```bash
+yq eval 'del(.a.a1)' sample.yml
+```
+will output
+```yaml
+a:
+  a2: frood
+```
+
 ## Delete entry in array
 Given a sample.yml file of:
 ```yaml
@@ -31,6 +48,21 @@ will output
 - 3
 ```
 
+## Delete nested entry in array
+Given a sample.yml file of:
+```yaml
+- a: cat
+  b: dog
+```
+then
+```bash
+yq eval 'del(.[0].a)' sample.yml
+```
+will output
+```yaml
+- b: dog
+```
+
 ## Delete no matches
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Traverse.md

@@ -106,7 +106,7 @@ b: *cat
 ```
 then
 ```bash
-yq eval '.b.[]' sample.yml
+yq eval '.b[]' sample.yml
 ```
 will output
 ```yaml
@@ -290,7 +290,7 @@ foobar:
 ```
 then
 ```bash
-yq eval '.foobar.[]' sample.yml
+yq eval '.foobar[]' sample.yml
 ```
 will output
 ```yaml
@@ -356,7 +356,7 @@ foobar:
 ```
 then
 ```bash
-yq eval '.foobarList.[]' sample.yml
+yq eval '.foobarList[]' sample.yml
 ```
 will output
 ```yaml
@@ -366,3 +366,21 @@ bar_thing
 foobarList_c
 ```
 
+## Select multiple indices
+Given a sample.yml file of:
+```yaml
+a:
+  - a
+  - b
+  - c
+```
+then
+```bash
+yq eval '.a[0, 2]' sample.yml
+```
+will output
+```yaml
+a
+c
+```
+

pkg/yqlib/doc/headers/Alternative (Default value).md

@@ -0,0 +1 @@
+This operator is used to provide alternative (or default) values when a particular expression is either null or false.

pkg/yqlib/doc/headers/Anchor and Alias Operators.md

@@ -0,0 +1,4 @@
+Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference aliases and remove anchor names).
+
+`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
+

pkg/yqlib/doc/headers/Explode.md

@@ -1 +0,0 @@
-Explodes (or dereferences) aliases and anchors.

pkg/yqlib/lib.go

@@ -36,9 +36,12 @@ var AssignAttributes = &OperationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Pre
 var AssignStyle = &OperationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: AssignStyleOperator}
 var AssignTag = &OperationType{Type: "ASSIGN_TAG", NumArgs: 2, Precedence: 40, Handler: AssignTagOperator}
 var AssignComment = &OperationType{Type: "ASSIGN_COMMENT", NumArgs: 2, Precedence: 40, Handler: AssignCommentsOperator}
+var AssignAnchor = &OperationType{Type: "ASSIGN_ANCHOR", NumArgs: 2, Precedence: 40, Handler: AssignAnchorOperator}
+var AssignAlias = &OperationType{Type: "ASSIGN_ALIAS", NumArgs: 2, Precedence: 40, Handler: AssignAliasOperator}
 
 var Multiply = &OperationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 45, Handler: MultiplyOperator}
 var Add = &OperationType{Type: "ADD", NumArgs: 2, Precedence: 45, Handler: AddOperator}
+var Alternative = &OperationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 45, Handler: AlternativeOperator}
 
 var Equals = &OperationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: EqualsOperator}
 var CreateMap = &OperationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 40, Handler: CreateMapOperator}
@@ -50,6 +53,8 @@ var Collect = &OperationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handle
 var GetStyle = &OperationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: GetStyleOperator}
 var GetTag = &OperationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: GetTagOperator}
 var GetComment = &OperationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: GetCommentsOperator}
+var GetAnchor = &OperationType{Type: "GET_ANCHOR", NumArgs: 0, Precedence: 50, Handler: GetAnchorOperator}
+var GetAlias = &OperationType{Type: "GET_ALIAS", NumArgs: 0, Precedence: 50, Handler: GetAliasOperator}
 var GetDocumentIndex = &OperationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: GetDocumentIndexOperator}
 var GetFilename = &OperationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: GetFilenameOperator}
 var GetFileIndex = &OperationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: GetFileIndexOperator}
@@ -60,6 +65,7 @@ var SortKeys = &OperationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Han
 
 var CollectObject = &OperationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: CollectObjectOperator}
 var TraversePath = &OperationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
+var TraverseArray = &OperationType{Type: "TRAVERSE_ARRAY", NumArgs: 1, Precedence: 50, Handler: TraverseArrayOperator}
 
 var DocumentFilter = &OperationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
 var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: SelfOperator}
@@ -72,6 +78,7 @@ var RecursiveDescent = &OperationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Pre
 var Select = &OperationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: SelectOperator}
 var Has = &OperationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: HasOperator}
 var DeleteChild = &OperationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: DeleteChildOperator}
+var DeleteImmediateChild = &OperationType{Type: "DELETE_IMMEDIATE_CHILD", NumArgs: 1, Precedence: 40, Handler: DeleteImmediateChildOperator}
 
 type Operation struct {
 	OperationType *OperationType

pkg/yqlib/operator_add.go

@@ -22,13 +22,7 @@ func AddAssignOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode
 	return d.GetMatchingNodes(matchingNodes, assignmentOpNode)
 }
 
-func toNodes(candidates *list.List) []*yaml.Node {
+func toNodes(candidate *CandidateNode) []*yaml.Node {
-
-	if candidates.Len() == 0 {
-		return []*yaml.Node{}
-	}
-	candidate := candidates.Front().Value.(*CandidateNode)
-
 	if candidate.Node.Tag == "!!null" {
 		return []*yaml.Node{}
 	}
@@ -44,27 +38,21 @@ func toNodes(candidates *list.List) []*yaml.Node {
 
 func AddOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
 	log.Debugf("Add operator")
-	var results = list.New()
-	lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
-	if err != nil {
-		return nil, err
-	}
-	rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
 
-	if err != nil {
+	return crossFunction(d, matchingNodes, pathNode, add)
-		return nil, err
+}
-	}
 
-	for el := lhs.Front(); el != nil; el = el.Next() {
+func add(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-		lhsCandidate := el.Value.(*CandidateNode)
+	lhs.Node = UnwrapDoc(lhs.Node)
-		lhsNode := UnwrapDoc(lhsCandidate.Node)
+	rhs.Node = UnwrapDoc(rhs.Node)
 
 	target := &CandidateNode{
-			Path:     lhsCandidate.Path,
+		Path:     lhs.Path,
-			Document: lhsCandidate.Document,
+		Document: lhs.Document,
-			Filename: lhsCandidate.Filename,
+		Filename: lhs.Filename,
 		Node:     &yaml.Node{},
 	}
+	lhsNode := lhs.Node
 
 	switch lhsNode.Kind {
 	case yaml.MappingNode:
@@ -74,10 +62,9 @@ func AddOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathT
 		target.Node.Style = lhsNode.Style
 		target.Node.Tag = "!!seq"
 		target.Node.Content = append(lhsNode.Content, toNodes(rhs)...)
-			results.PushBack(target)
 	case yaml.ScalarNode:
 		return nil, fmt.Errorf("Scalars not yet supported for addition")
 	}
-	}
+
-	return results, nil
+	return target, nil
 }

pkg/yqlib/operator_add_test.go

@@ -21,6 +21,14 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!seq)::[1, 2, 3, 4]\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		expression: `[1] + ([2], [3])`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 1\n- 2\n",
+			"D0, P[], (!!seq)::- 1\n- 3\n",
+		},
+	},
 	{
 		description: "Concatenate null to array",
 		document:    `{a: [1,2]}`,

pkg/yqlib/operator_alternative.go

@@ -0,0 +1,28 @@
+package yqlib
+
+import (
+	"container/list"
+)
+
+// corssFunction no matches
+// can boolean use crossfunction
+
+func AlternativeOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+	log.Debugf("-- alternative")
+	return crossFunction(d, matchingNodes, pathNode, alternativeFunc)
+}
+
+func alternativeFunc(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	lhs.Node = UnwrapDoc(lhs.Node)
+	rhs.Node = UnwrapDoc(rhs.Node)
+	log.Debugf("Alternative LHS: %v", lhs.Node.Tag)
+	log.Debugf("-          RHS: %v", rhs.Node.Tag)
+
+	isTrue, err := isTruthy(lhs)
+	if err != nil {
+		return nil, err
+	} else if isTrue {
+		return lhs, nil
+	}
+	return rhs, nil
+}

pkg/yqlib/operator_alternative_test.go

@@ -0,0 +1,62 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var alternativeOperatorScenarios = []expressionScenario{
+	{
+		description: "LHS is defined",
+		expression:  `.a // "hello"`,
+		document:    `{a: bridge}`,
+		expected: []string{
+			"D0, P[a], (!!str)::bridge\n",
+		},
+	},
+	{
+		description: "LHS is not defined",
+		expression:  `.a // "hello"`,
+		document:    `{}`,
+		expected: []string{
+			"D0, P[], (!!str)::hello\n",
+		},
+	},
+	{
+		description: "LHS is null",
+		expression:  `.a // "hello"`,
+		document:    `{a: ~}`,
+		expected: []string{
+			"D0, P[], (!!str)::hello\n",
+		},
+	},
+	{
+		description: "LHS is false",
+		expression:  `.a // "hello"`,
+		document:    `{a: false}`,
+		expected: []string{
+			"D0, P[], (!!str)::hello\n",
+		},
+	},
+	{
+		description: "RHS is an expression",
+		expression:  `.a // .b`,
+		document:    `{a: false, b: cat}`,
+		expected: []string{
+			"D0, P[b], (!!str)::cat\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `false // true`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+}
+
+func TestAlternativeOperatorScenarios(t *testing.T) {
+	for _, tt := range alternativeOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Alternative (Default value)", alternativeOperatorScenarios)
+}

pkg/yqlib/operator_anchors_aliases.go

@@ -6,6 +6,88 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+func AssignAliasOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+
+	log.Debugf("AssignAlias operator!")
+
+	rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+	if err != nil {
+		return nil, err
+	}
+	aliasName := ""
+	if rhs.Front() != nil {
+		aliasName = rhs.Front().Value.(*CandidateNode).Node.Value
+	}
+
+	lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
+
+	if err != nil {
+		return nil, err
+	}
+
+	for el := lhs.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		log.Debugf("Setting aliasName : %v", candidate.GetKey())
+		candidate.Node.Kind = yaml.AliasNode
+		candidate.Node.Value = aliasName
+	}
+	return matchingNodes, nil
+}
+
+func GetAliasOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+	log.Debugf("GetAlias operator!")
+	var results = list.New()
+
+	for el := matchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		node := &yaml.Node{Kind: yaml.ScalarNode, Value: candidate.Node.Value, Tag: "!!str"}
+		lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
+		results.PushBack(lengthCand)
+	}
+	return results, nil
+}
+
+func AssignAnchorOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+
+	log.Debugf("AssignAnchor operator!")
+
+	rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+	if err != nil {
+		return nil, err
+	}
+	anchorName := ""
+	if rhs.Front() != nil {
+		anchorName = rhs.Front().Value.(*CandidateNode).Node.Value
+	}
+
+	lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
+
+	if err != nil {
+		return nil, err
+	}
+
+	for el := lhs.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		log.Debugf("Setting anchorName of : %v", candidate.GetKey())
+		candidate.Node.Anchor = anchorName
+	}
+	return matchingNodes, nil
+}
+
+func GetAnchorOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+	log.Debugf("GetAnchor operator!")
+	var results = list.New()
+
+	for el := matchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		anchor := candidate.Node.Anchor
+		node := &yaml.Node{Kind: yaml.ScalarNode, Value: anchor, Tag: "!!str"}
+		lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
+		results.PushBack(lengthCand)
+	}
+	return results, nil
+}
+
 func ExplodeOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
 	log.Debugf("-- ExplodeOperation")
 

pkg/yqlib/operator_anchors_aliases_test.go

@@ -4,7 +4,39 @@ import (
 	"testing"
 )
 
-var explodeTest = []expressionScenario{
+var anchorOperatorScenarios = []expressionScenario{
+	{
+		description: "Get anchor",
+		document:    `a: &billyBob cat`,
+		expression:  `.a | anchor`,
+		expected: []string{
+			"D0, P[a], (!!str)::billyBob\n",
+		},
+	},
+	{
+		description: "Set anchor",
+		document:    `a: cat`,
+		expression:  `.a anchor = "foobar"`,
+		expected: []string{
+			"D0, P[], (doc)::a: &foobar cat\n",
+		},
+	},
+	{
+		description: "Get alias",
+		document:    `{b: &billyBob meow, a: *billyBob}`,
+		expression:  `.a | alias`,
+		expected: []string{
+			"D0, P[a], (!!str)::billyBob\n",
+		},
+	},
+	{
+		description: "Set alias",
+		document:    `{b: &meow purr, a: cat}`,
+		expression:  `.a alias = "meow"`,
+		expected: []string{
+			"D0, P[], (doc)::{b: &meow purr, a: *meow}\n",
+		},
+	},
 	{
 		description: "Explode alias and anchor",
 		document:    `{f : {a: &a cat, b: *a}}`,
@@ -82,9 +114,9 @@ foobar:
 	},
 }
 
-func TestExplodeOperatorScenarios(t *testing.T) {
+func TestAnchorAliaseOperatorScenarios(t *testing.T) {
-	for _, tt := range explodeTest {
+	for _, tt := range anchorOperatorScenarios {
 		testScenario(t, &tt)
 	}
-	documentScenarios(t, "Explode", explodeTest)
+	documentScenarios(t, "Anchor and Alias Operators", anchorOperatorScenarios)
 }

pkg/yqlib/operator_assign_test.go

@@ -80,6 +80,14 @@ var assignOperatorScenarios = []expressionScenario{
 	{
 		description: "Update selected results",
 		document:    `{a: {b: apple, c: cactus}}`,
+		expression:  `(.a[] | select(. == "apple")) = "frog"`,
+		expected: []string{
+			"D0, P[], (doc)::{a: {b: frog, c: cactus}}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: {b: apple, c: cactus}}`,
 		expression: `(.a.[] | select(. == "apple")) = "frog"`,
 		expected: []string{
 			"D0, P[], (doc)::{a: {b: frog, c: cactus}}\n",

pkg/yqlib/operator_booleans.go

@@ -25,74 +25,39 @@ func isTruthy(c *CandidateNode) (bool, error) {
 
 type boolOp func(bool, bool) bool
 
-func performBoolOp(results *list.List, lhs *list.List, rhs *list.List, op boolOp) error {
+func performBoolOp(op boolOp) func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-	for lhsChild := lhs.Front(); lhsChild != nil; lhsChild = lhsChild.Next() {
+	return func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-		lhsCandidate := lhsChild.Value.(*CandidateNode)
+		lhs.Node = UnwrapDoc(lhs.Node)
-		lhsTrue, errDecoding := isTruthy(lhsCandidate)
+		rhs.Node = UnwrapDoc(rhs.Node)
+
+		lhsTrue, errDecoding := isTruthy(lhs)
 		if errDecoding != nil {
-			return errDecoding
+			return nil, errDecoding
 		}
 
-		for rhsChild := rhs.Front(); rhsChild != nil; rhsChild = rhsChild.Next() {
+		rhsTrue, errDecoding := isTruthy(rhs)
-			rhsCandidate := rhsChild.Value.(*CandidateNode)
-			rhsTrue, errDecoding := isTruthy(rhsCandidate)
 		if errDecoding != nil {
-				return errDecoding
+			return nil, errDecoding
-			}
-			boolResult := createBooleanCandidate(lhsCandidate, op(lhsTrue, rhsTrue))
-			results.PushBack(boolResult)
-		}
-	}
-	return nil
-}
-
-func booleanOp(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode, op boolOp) (*list.List, error) {
-	var results = list.New()
-
-	if matchingNodes.Len() == 0 {
-		lhs, err := d.GetMatchingNodes(list.New(), pathNode.Lhs)
-		if err != nil {
-			return nil, err
-		}
-		rhs, err := d.GetMatchingNodes(list.New(), pathNode.Rhs)
-		if err != nil {
-			return nil, err
-		}
-		return results, performBoolOp(results, lhs, rhs, op)
 		}
 
-	for el := matchingNodes.Front(); el != nil; el = el.Next() {
+		return createBooleanCandidate(lhs, op(lhsTrue, rhsTrue)), nil
-		candidate := el.Value.(*CandidateNode)
-		lhs, err := d.GetMatchingNodes(nodeToMap(candidate), pathNode.Lhs)
-		if err != nil {
-			return nil, err
 	}
-		rhs, err := d.GetMatchingNodes(nodeToMap(candidate), pathNode.Rhs)
-		if err != nil {
-			return nil, err
-		}
-
-		err = performBoolOp(results, lhs, rhs, op)
-		if err != nil {
-			return nil, err
-		}
-
-	}
-	return results, nil
 }
 
 func OrOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
 	log.Debugf("-- orOp")
-	return booleanOp(d, matchingNodes, pathNode, func(b1 bool, b2 bool) bool {
+	return crossFunction(d, matchingNodes, pathNode, performBoolOp(
+		func(b1 bool, b2 bool) bool {
 			return b1 || b2
-	})
+		}))
 }
 
 func AndOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
 	log.Debugf("-- AndOp")
-	return booleanOp(d, matchingNodes, pathNode, func(b1 bool, b2 bool) bool {
+	return crossFunction(d, matchingNodes, pathNode, performBoolOp(
+		func(b1 bool, b2 bool) bool {
 			return b1 && b2
-	})
+		}))
 }
 
 func NotOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {

pkg/yqlib/operator_collect_object.go

@@ -55,7 +55,7 @@ func CollectObjectOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *
 func createChildCandidate(candidate *CandidateNode, index int) *CandidateNode {
 	return &CandidateNode{
 		Document: candidate.Document,
-		Path:     append(candidate.Path, index),
+		Path:     candidate.CreateChildPath(index),
 		Filename: candidate.Filename,
 		Node:     candidate.Node.Content[index],
 	}

pkg/yqlib/operator_delete.go

@@ -2,39 +2,67 @@ package yqlib
 
 import (
 	"container/list"
+	"fmt"
 
 	yaml "gopkg.in/yaml.v3"
 )
 
 func DeleteChildOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
-	// for each lhs, splat the node,
-	// the intersect it against the rhs expression
-	// recreate the contents using only the intersection result.
 
-	for el := matchingNodes.Front(); el != nil; el = el.Next() {
+	nodesToDelete, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
-		candidate := el.Value.(*CandidateNode)
+
-		elMap := list.New()
-		elMap.PushBack(candidate)
-		nodesToDelete, err := d.GetMatchingNodes(elMap, pathNode.Rhs)
-		log.Debug("nodesToDelete:\n%v", NodesToString(nodesToDelete))
 	if err != nil {
 		return nil, err
 	}
 
-		realNode := UnwrapDoc(candidate.Node)
+	for el := nodesToDelete.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
 
-		if realNode.Kind == yaml.SequenceNode {
+		deleteImmediateChildOp := &Operation{
-			deleteFromArray(candidate, nodesToDelete)
+			OperationType: DeleteImmediateChild,
-		} else if realNode.Kind == yaml.MappingNode {
+			Value:         candidate.Path[len(candidate.Path)-1],
-			deleteFromMap(candidate, nodesToDelete)
+		}
-		} else {
+
-			log.Debug("Cannot delete from node that's not a map or array %v", NodeToString(candidate))
+		deleteImmediateChildOpNode := &PathTreeNode{
+			Operation: deleteImmediateChildOp,
+			Rhs:       createTraversalTree(candidate.Path[0 : len(candidate.Path)-1]),
+		}
+
+		_, err := d.GetMatchingNodes(matchingNodes, deleteImmediateChildOpNode)
+		if err != nil {
+			return nil, err
 		}
 	}
 	return matchingNodes, nil
 }
 
-func deleteFromMap(candidate *CandidateNode, nodesToDelete *list.List) {
+func DeleteImmediateChildOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
+	parents, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+
+	if err != nil {
+		return nil, err
+	}
+
+	childPath := pathNode.Operation.Value
+
+	log.Debug("childPath to remove %v", childPath)
+
+	for el := parents.Front(); el != nil; el = el.Next() {
+		parent := el.Value.(*CandidateNode)
+		parentNode := UnwrapDoc(parent.Node)
+		if parentNode.Kind == yaml.MappingNode {
+			deleteFromMap(parent, childPath)
+		} else if parentNode.Kind == yaml.SequenceNode {
+			deleteFromArray(parent, childPath)
+		} else {
+			return nil, fmt.Errorf("Cannot delete nodes from parent of tag %v", parentNode.Tag)
+		}
+
+	}
+	return matchingNodes, nil
+}
+
+func deleteFromMap(candidate *CandidateNode, childPath interface{}) {
 	log.Debug("deleteFromMap")
 	node := UnwrapDoc(candidate.Node)
 	contents := node.Content
@@ -47,15 +75,10 @@ func deleteFromMap(candidate *CandidateNode, nodesToDelete *list.List) {
 		childCandidate := &CandidateNode{
 			Node:     value,
 			Document: candidate.Document,
-			Path:     append(candidate.Path, key.Value),
+			Path:     candidate.CreateChildPath(key.Value),
 		}
 
-		shouldDelete := false
+		shouldDelete := key.Value == childPath
-		for el := nodesToDelete.Front(); el != nil && !shouldDelete; el = el.Next() {
-			if el.Value.(*CandidateNode).GetKey() == childCandidate.GetKey() {
-				shouldDelete = true
-			}
-		}
 
 		log.Debugf("shouldDelete %v ? %v", childCandidate.GetKey(), shouldDelete)
 
@@ -66,7 +89,7 @@ func deleteFromMap(candidate *CandidateNode, nodesToDelete *list.List) {
 	node.Content = newContents
 }
 
-func deleteFromArray(candidate *CandidateNode, nodesToDelete *list.List) {
+func deleteFromArray(candidate *CandidateNode, childPath interface{}) {
 	log.Debug("deleteFromArray")
 	node := UnwrapDoc(candidate.Node)
 	contents := node.Content
@@ -75,18 +98,7 @@ func deleteFromArray(candidate *CandidateNode, nodesToDelete *list.List) {
 	for index := 0; index < len(contents); index = index + 1 {
 		value := contents[index]
 
-		childCandidate := &CandidateNode{
+		shouldDelete := fmt.Sprintf("%v", index) == fmt.Sprintf("%v", childPath)
-			Node:     value,
-			Document: candidate.Document,
-			Path:     append(candidate.Path, index),
-		}
-
-		shouldDelete := false
-		for el := nodesToDelete.Front(); el != nil && !shouldDelete; el = el.Next() {
-			if el.Value.(*CandidateNode).GetKey() == childCandidate.GetKey() {
-				shouldDelete = true
-			}
-		}
 
 		if !shouldDelete {
 			newContents = append(newContents, value)

pkg/yqlib/operator_delete_test.go

@@ -13,6 +13,22 @@ var deleteOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: cat}\n",
 		},
 	},
+	{
+		description: "Delete nested entry in map",
+		document:    `{a: {a1: fred, a2: frood}}`,
+		expression:  `del(.a.a1)`,
+		expected: []string{
+			"D0, P[], (doc)::{a: {a2: frood}}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: {a1: fred, a2: frood}}`,
+		expression: `del(.. | select(.=="frood"))`,
+		expected: []string{
+			"D0, P[], (!!map)::{a: {a1: fred}}\n",
+		},
+	},
 	{
 		description: "Delete entry in array",
 		document:    `[1,2,3]`,
@@ -21,6 +37,14 @@ var deleteOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::[1, 3]\n",
 		},
 	},
+	{
+		description: "Delete nested entry in array",
+		document:    `[{a: cat, b: dog}]`,
+		expression:  `del(.[0].a)`,
+		expected: []string{
+			"D0, P[], (doc)::[{b: dog}]\n",
+		},
+	},
 	{
 		description: "Delete no matches",
 		document:    `{a: cat, b: dog}`,

pkg/yqlib/operator_multiply.go

@@ -104,19 +104,6 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode,
 	return lhs, nil
 }
 
-func createTraversalTree(path []interface{}) *PathTreeNode {
-	if len(path) == 0 {
-		return &PathTreeNode{Operation: &Operation{OperationType: SelfReference}}
-	} else if len(path) == 1 {
-		return &PathTreeNode{Operation: &Operation{OperationType: TraversePath, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
-	}
-	return &PathTreeNode{
-		Operation: &Operation{OperationType: ShortPipe},
-		Lhs:       createTraversalTree(path[0:1]),
-		Rhs:       createTraversalTree(path[1:])}
-
-}
-
 func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) error {
 
 	log.Debugf("merge - applyAssignment lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs))

pkg/yqlib/operator_path_test.go

@@ -13,6 +13,23 @@ var pathOperatorScenarios = []expressionScenario{
 			"D0, P[a b], (!!seq)::- a\n- b\n",
 		},
 	},
+	{
+		skipDoc: true,
+		document: `a:
+  b:
+    c:
+    - 0
+    - 1
+    - 2
+    - 3`,
+		expression: `.a.b.c.[]`,
+		expected: []string{
+			"D0, P[a b c 0], (!!int)::0\n",
+			"D0, P[a b c 1], (!!int)::1\n",
+			"D0, P[a b c 2], (!!int)::2\n",
+			"D0, P[a b c 3], (!!int)::3\n",
+		},
+	},
 	{
 		description: "Get map key",
 		document:    `{a: {b: cat}}`,

pkg/yqlib/operator_traverse_path.go

@@ -1,13 +1,12 @@
 package yqlib
 
 import (
-	"fmt"
-
 	"container/list"
+	"fmt"
+	"strconv"
 
 	"github.com/elliotchance/orderedmap"
-
+	yaml "gopkg.in/yaml.v3"
-	"gopkg.in/yaml.v3"
 )
 
 type TraversePreferences struct {
@@ -15,32 +14,25 @@ type TraversePreferences struct {
 }
 
 func Splat(d *dataTreeNavigator, matches *list.List) (*list.List, error) {
-	preferences := &TraversePreferences{DontFollowAlias: true}
+	return traverseNodesWithArrayIndices(matches, make([]*yaml.Node, 0), false)
-	splatOperation := &Operation{OperationType: TraversePath, Value: "[]", Preferences: preferences}
-	splatTreeNode := &PathTreeNode{Operation: splatOperation}
-	return TraversePathOperator(d, matches, splatTreeNode)
 }
 
 func TraversePathOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
 	log.Debugf("-- Traversing")
 	var matchingNodeMap = list.New()
-	var newNodes []*CandidateNode
-	var err error
 
 	for el := matchMap.Front(); el != nil; el = el.Next() {
-		newNodes, err = traverse(d, el.Value.(*CandidateNode), pathNode.Operation)
+		newNodes, err := traverse(d, el.Value.(*CandidateNode), pathNode.Operation)
 		if err != nil {
 			return nil, err
 		}
-		for _, n := range newNodes {
+		matchingNodeMap.PushBackList(newNodes)
-			matchingNodeMap.PushBack(n)
-		}
 	}
 
 	return matchingNodeMap, nil
 }
 
-func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Operation) ([]*CandidateNode, error) {
+func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Operation) (*list.List, error) {
 	log.Debug("Traversing %v", NodeToString(matchingNode))
 	value := matchingNode.Node
 
@@ -61,34 +53,12 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Oper
 	switch value.Kind {
 	case yaml.MappingNode:
 		log.Debug("its a map with %v entries", len(value.Content)/2)
-		var newMatches = orderedmap.NewOrderedMap()
+		followAlias := true
-		err := traverseMap(newMatches, matchingNode, operation)
 
-		if err != nil {
+		if operation.Preferences != nil {
-			return nil, err
+			followAlias = !operation.Preferences.(*TraversePreferences).DontFollowAlias
 		}
-
+		return traverseMap(matchingNode, operation.StringValue, followAlias, false)
-		if newMatches.Len() == 0 {
-			//no matches, create one automagically
-			valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}
-			node := matchingNode.Node
-			node.Content = append(node.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: operation.StringValue}, valueNode)
-			candidateNode := &CandidateNode{
-				Node:     valueNode,
-				Path:     append(matchingNode.Path, operation.StringValue),
-				Document: matchingNode.Document,
-			}
-			newMatches.Set(candidateNode.GetKey(), candidateNode)
-
-		}
-
-		arrayMatches := make([]*CandidateNode, newMatches.Len())
-		i := 0
-		for el := newMatches.Front(); el != nil; el = el.Next() {
-			arrayMatches[i] = el.Value.(*CandidateNode)
-			i++
-		}
-		return arrayMatches, nil
 
 	case yaml.SequenceNode:
 		log.Debug("its a sequence of %v things!", len(value.Content))
@@ -102,104 +72,117 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Oper
 		log.Debug("digging into doc node")
 		return traverse(d, &CandidateNode{
 			Node:      matchingNode.Node.Content[0],
+			Filename:  matchingNode.Filename,
+			FileIndex: matchingNode.FileIndex,
 			Document:  matchingNode.Document}, operation)
 	default:
-		return nil, nil
+		return list.New(), nil
 	}
 }
 
-func keyMatches(key *yaml.Node, pathNode *Operation) bool {
+func TraverseArrayOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
-	return pathNode.Value == "[]" || Match(key.Value, pathNode.StringValue)
+	// rhs is a collect expression that will yield indexes to retreive of the arrays
-}
 
-func traverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, operation *Operation) error {
+	rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
-	// value.Content is a concatenated array of key, value,
-	// so keys are in the even indexes, values in odd.
-	// merge aliases are defined first, but we only want to traverse them
-	// if we don't find a match directly on this node first.
-	//TODO ALIASES, auto creation?
-
-	node := candidate.Node
-
-	followAlias := true
-
-	if operation.Preferences != nil {
-		followAlias = !operation.Preferences.(*TraversePreferences).DontFollowAlias
-	}
-
-	var contents = node.Content
-	for index := 0; index < len(contents); index = index + 2 {
-		key := contents[index]
-		value := contents[index+1]
-
-		log.Debug("checking %v (%v)", key.Value, key.Tag)
-		//skip the 'merge' tag, find a direct match first
-		if key.Tag == "!!merge" && followAlias {
-			log.Debug("Merge anchor")
-			err := traverseMergeAnchor(newMatches, candidate, value, operation)
 	if err != nil {
-				return err
+		return nil, err
-			}
-		} else if keyMatches(key, operation) {
-			log.Debug("MATCHED")
-			candidateNode := &CandidateNode{
-				Node:     value,
-				Path:     append(candidate.Path, key.Value),
-				Document: candidate.Document,
-			}
-			newMatches.Set(candidateNode.GetKey(), candidateNode)
-		}
 	}
 
-	return nil
+	var indicesToTraverse = rhs.Front().Value.(*CandidateNode).Node.Content
+
+	return traverseNodesWithArrayIndices(matchingNodes, indicesToTraverse, true)
 }
 
-func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, operation *Operation) error {
+func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, followAlias bool) (*list.List, error) {
-	switch value.Kind {
+	var matchingNodeMap = list.New()
-	case yaml.AliasNode:
+	for el := matchingNodes.Front(); el != nil; el = el.Next() {
-		candidateNode := &CandidateNode{
+		candidate := el.Value.(*CandidateNode)
-			Node:     value.Alias,
+		newNodes, err := traverseArrayIndices(candidate, indicesToTraverse, followAlias)
-			Path:     originalCandidate.Path,
-			Document: originalCandidate.Document,
-		}
-		return traverseMap(newMatches, candidateNode, operation)
-	case yaml.SequenceNode:
-		for _, childValue := range value.Content {
-			err := traverseMergeAnchor(newMatches, originalCandidate, childValue, operation)
 		if err != nil {
-				return err
+			return nil, err
 		}
+		matchingNodeMap.PushBackList(newNodes)
 	}
-	}
+
-	return nil
+	return matchingNodeMap, nil
 }
 
-func traverseArray(candidate *CandidateNode, operation *Operation) ([]*CandidateNode, error) {
+func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, followAlias bool) (*list.List, error) { // call this if doc / alias like the other traverse
-	log.Debug("operation Value %v", operation.Value)
+	node := matchingNode.Node
-	if operation.Value == "[]" {
+	if node.Tag == "!!null" {
+		log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
+		// auto vivification, make it into an empty array
+		node.Tag = ""
+		node.Kind = yaml.SequenceNode
+	}
 
-		var contents = candidate.Node.Content
+	if node.Kind == yaml.AliasNode {
-		var newMatches = make([]*CandidateNode, len(contents))
+		matchingNode.Node = node.Alias
+		return traverseArrayIndices(matchingNode, indicesToTraverse, followAlias)
+	} else if node.Kind == yaml.SequenceNode {
+		return traverseArrayWithIndices(matchingNode, indicesToTraverse)
+	} else if node.Kind == yaml.MappingNode {
+		return traverseMapWithIndices(matchingNode, indicesToTraverse, followAlias)
+	} else if node.Kind == yaml.DocumentNode {
+		return traverseArrayIndices(&CandidateNode{
+			Node:      matchingNode.Node.Content[0],
+			Filename:  matchingNode.Filename,
+			FileIndex: matchingNode.FileIndex,
+			Document:  matchingNode.Document}, indicesToTraverse, followAlias)
+	}
+	log.Debugf("OperatorArrayTraverse skipping %v as its a %v", matchingNode, node.Tag)
+	return list.New(), nil
+}
+
+func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, followAlias bool) (*list.List, error) {
+	if len(indices) == 0 {
+		return traverseMap(candidate, "", followAlias, true)
+	}
+
+	var matchingNodeMap = list.New()
+
+	for _, indexNode := range indices {
+		log.Debug("traverseMapWithIndices: %v", indexNode.Value)
+		newNodes, err := traverseMap(candidate, indexNode.Value, followAlias, false)
+		if err != nil {
+			return nil, err
+		}
+		matchingNodeMap.PushBackList(newNodes)
+	}
+
+	return matchingNodeMap, nil
+}
+
+func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*list.List, error) {
+	log.Debug("traverseArrayWithIndices")
+	var newMatches = list.New()
+	node := UnwrapDoc(candidate.Node)
+	if len(indices) == 0 {
+		log.Debug("splatting")
 		var index int64
-		for index = 0; index < int64(len(contents)); index = index + 1 {
+		for index = 0; index < int64(len(node.Content)); index = index + 1 {
-			newMatches[index] = &CandidateNode{
+
+			newMatches.PushBack(&CandidateNode{
 				Document: candidate.Document,
-				Path:     append(candidate.Path, index),
+				Path:     candidate.CreateChildPath(index),
-				Node:     contents[index],
+				Node:     node.Content[index],
-			}
+			})
 		}
 		return newMatches, nil
 
 	}
 
-	switch operation.Value.(type) {
+	for _, indexNode := range indices {
-	case int64:
+		log.Debug("traverseArrayWithIndices: '%v'", indexNode.Value)
-		index := operation.Value.(int64)
+		index, err := strconv.ParseInt(indexNode.Value, 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("Cannot index array with '%v' (%v)", indexNode.Value, err)
+		}
 		indexToUse := index
-		contentLength := int64(len(candidate.Node.Content))
+		contentLength := int64(len(node.Content))
 		for contentLength <= index {
-			candidate.Node.Content = append(candidate.Node.Content, &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"})
+			node.Content = append(node.Content, &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"})
-			contentLength = int64(len(candidate.Node.Content))
+			contentLength = int64(len(node.Content))
 		}
 
 		if indexToUse < 0 {
@@ -210,14 +193,107 @@ func traverseArray(candidate *CandidateNode, operation *Operation) ([]*Candidate
 			return nil, fmt.Errorf("Index [%v] out of range, array size is %v", index, contentLength)
 		}
 
-		return []*CandidateNode{&CandidateNode{
+		newMatches.PushBack(&CandidateNode{
-			Node:     candidate.Node.Content[indexToUse],
+			Node:     node.Content[indexToUse],
 			Document: candidate.Document,
-			Path:     append(candidate.Path, index),
+			Path:     candidate.CreateChildPath(index),
-		}}, nil
+		})
-	default:
+	}
-		log.Debug("argument not an int (%v), no array matches", operation.Value)
+	return newMatches, nil
-		return nil, nil
+}
+
+func keyMatches(key *yaml.Node, wantedKey string) bool {
+	return Match(key.Value, wantedKey)
+}
+
+func traverseMap(matchingNode *CandidateNode, key string, followAlias bool, splat bool) (*list.List, error) {
+	var newMatches = orderedmap.NewOrderedMap()
+	err := doTraverseMap(newMatches, matchingNode, key, followAlias, splat)
+
+	if err != nil {
+		return nil, err
 	}
 
+	if newMatches.Len() == 0 {
+		//no matches, create one automagically
+		valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}
+		node := matchingNode.Node
+		node.Content = append(node.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: key}, valueNode)
+		candidateNode := &CandidateNode{
+			Node:     valueNode,
+			Path:     append(matchingNode.Path, key),
+			Document: matchingNode.Document,
+		}
+		newMatches.Set(candidateNode.GetKey(), candidateNode)
+
+	}
+
+	results := list.New()
+	i := 0
+	for el := newMatches.Front(); el != nil; el = el.Next() {
+		results.PushBack(el.Value)
+		i++
+	}
+	return results, nil
+}
+
+func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, followAlias bool, splat bool) error {
+	// value.Content is a concatenated array of key, value,
+	// so keys are in the even indexes, values in odd.
+	// merge aliases are defined first, but we only want to traverse them
+	// if we don't find a match directly on this node first.
+
+	node := candidate.Node
+
+	var contents = node.Content
+	for index := 0; index < len(contents); index = index + 2 {
+		key := contents[index]
+		value := contents[index+1]
+
+		log.Debug("checking %v (%v)", key.Value, key.Tag)
+		//skip the 'merge' tag, find a direct match first
+		if key.Tag == "!!merge" && followAlias {
+			log.Debug("Merge anchor")
+			err := traverseMergeAnchor(newMatches, candidate, value, wantedKey, splat)
+			if err != nil {
+				return err
+			}
+		} else if splat || keyMatches(key, wantedKey) {
+			log.Debug("MATCHED")
+			candidateNode := &CandidateNode{
+				Node:     value,
+				Path:     candidate.CreateChildPath(key.Value),
+				Document: candidate.Document,
+			}
+			newMatches.Set(candidateNode.GetKey(), candidateNode)
+		}
+	}
+
+	return nil
+}
+
+func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, splat bool) error {
+	switch value.Kind {
+	case yaml.AliasNode:
+		candidateNode := &CandidateNode{
+			Node:     value.Alias,
+			Path:     originalCandidate.Path,
+			Document: originalCandidate.Document,
+		}
+		return doTraverseMap(newMatches, candidateNode, wantedKey, true, splat)
+	case yaml.SequenceNode:
+		for _, childValue := range value.Content {
+			err := traverseMergeAnchor(newMatches, originalCandidate, childValue, wantedKey, splat)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func traverseArray(candidate *CandidateNode, operation *Operation) (*list.List, error) {
+	log.Debug("operation Value %v", operation.Value)
+	indices := []*yaml.Node{&yaml.Node{Value: operation.StringValue}}
+	return traverseArrayWithIndices(candidate, indices)
 }

pkg/yqlib/operator_traverse_path_test.go

@@ -65,7 +65,7 @@ var traversePathOperatorScenarios = []expressionScenario{
 	},
 	{
 		skipDoc:    true,
-		document:   `{}`,
+		document:   ``,
 		expression: `.[1].a`,
 		expected: []string{
 			"D0, P[1 a], (!!null)::null\n",
@@ -137,6 +137,14 @@ var traversePathOperatorScenarios = []expressionScenario{
 	{
 		description: "Traversing aliases with splat",
 		document:    `{a: &cat {c: frog}, b: *cat}`,
+		expression:  `.b[]`,
+		expected: []string{
+			"D0, P[b c], (!!str)::frog\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: &cat {c: frog}, b: *cat}`,
 		expression: `.b.[]`,
 		expected: []string{
 			"D0, P[b c], (!!str)::frog\n",
@@ -150,12 +158,6 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[b c], (!!str)::frog\n",
 		},
 	},
-	{
-		skipDoc:    true,
-		document:   `[1,2,3]`,
-		expression: `.b`,
-		expected:   []string{},
-	},
 	{
 		description: "Traversing arrays by index",
 		document:    `[1,2,3]`,
@@ -215,6 +217,16 @@ var traversePathOperatorScenarios = []expressionScenario{
 	{
 		description: "Splatting merge anchors",
 		document:    mergeDocSample,
+		expression:  `.foobar[]`,
+		expected: []string{
+			"D0, P[foobar c], (!!str)::foo_c\n",
+			"D0, P[foobar a], (!!str)::foo_a\n",
+			"D0, P[foobar thing], (!!str)::foobar_thing\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   mergeDocSample,
 		expression: `.foobar.[]`,
 		expected: []string{
 			"D0, P[foobar c], (!!str)::foo_c\n",
@@ -266,6 +278,17 @@ var traversePathOperatorScenarios = []expressionScenario{
 	{
 		description: "Splatting merge anchor lists",
 		document:    mergeDocSample,
+		expression:  `.foobarList[]`,
+		expected: []string{
+			"D0, P[foobarList b], (!!str)::bar_b\n",
+			"D0, P[foobarList a], (!!str)::foo_a\n",
+			"D0, P[foobarList thing], (!!str)::bar_thing\n",
+			"D0, P[foobarList c], (!!str)::foobarList_c\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   mergeDocSample,
 		expression: `.foobarList.[]`,
 		expected: []string{
 			"D0, P[foobarList b], (!!str)::bar_b\n",
@@ -274,6 +297,112 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[foobarList c], (!!str)::foobarList_c\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `[a,b,c]`,
+		expression: `.[]`,
+		expected: []string{
+			"D0, P[0], (!!str)::a\n",
+			"D0, P[1], (!!str)::b\n",
+			"D0, P[2], (!!str)::c\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[a,b,c]`,
+		expression: `[]`,
+		expected: []string{
+			"D0, P[], (!!seq)::[]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [a,b,c]}`,
+		expression: `.a[0]`,
+		expected: []string{
+			"D0, P[a 0], (!!str)::a\n",
+		},
+	},
+	{
+		description: "Select multiple indices",
+		document:    `{a: [a,b,c]}`,
+		expression:  `.a[0, 2]`,
+		expected: []string{
+			"D0, P[a 0], (!!str)::a\n",
+			"D0, P[a 2], (!!str)::c\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [a,b,c]}`,
+		expression: `.a.[0, 2]`,
+		expected: []string{
+			"D0, P[a 0], (!!str)::a\n",
+			"D0, P[a 2], (!!str)::c\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [a,b,c]}`,
+		expression: `.a[-1]`,
+		expected: []string{
+			"D0, P[a -1], (!!str)::c\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [a,b,c]}`,
+		expression: `.a.[-1]`,
+		expected: []string{
+			"D0, P[a -1], (!!str)::c\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [a,b,c]}`,
+		expression: `.a[-2]`,
+		expected: []string{
+			"D0, P[a -2], (!!str)::b\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [a,b,c]}`,
+		expression: `.a.[-2]`,
+		expected: []string{
+			"D0, P[a -2], (!!str)::b\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [a,b,c]}`,
+		expression: `.a[]`,
+		expected: []string{
+			"D0, P[a 0], (!!str)::a\n",
+			"D0, P[a 1], (!!str)::b\n",
+			"D0, P[a 2], (!!str)::c\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [a,b,c]}`,
+		expression: `.a.[]`,
+		expected: []string{
+			"D0, P[a 0], (!!str)::a\n",
+			"D0, P[a 1], (!!str)::b\n",
+			"D0, P[a 2], (!!str)::c\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [a,b,c]}`,
+		expression: `.a | .[]`,
+		expected: []string{
+			"D0, P[a 0], (!!str)::a\n",
+			"D0, P[a 1], (!!str)::b\n",
+			"D0, P[a 2], (!!str)::c\n",
+		},
+	},
 }
 
 func TestTraversePathOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_value_test.go

@@ -36,13 +36,22 @@ var valueOperatorScenarios = []expressionScenario{
 		expected: []string{
 			"D0, P[], (!!float)::5e-10\n",
 		},
-	}, {
+	},
+	{
 		document:   ``,
 		expression: `"cat"`,
 		expected: []string{
 			"D0, P[], (!!str)::cat\n",
 		},
-	}, {
+	},
+	{
+		document:   ``,
+		expression: `"frog jumps"`,
+		expected: []string{
+			"D0, P[], (!!str)::frog jumps\n",
+		},
+	},
+	{
 		document:   ``,
 		expression: `"1.3"`,
 		expected: []string{

pkg/yqlib/operators.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"container/list"
+	"fmt"
 
 	"gopkg.in/yaml.v3"
 )
@@ -33,3 +34,15 @@ func nodeToMap(candidate *CandidateNode) *list.List {
 	elMap.PushBack(candidate)
 	return elMap
 }
+
+func createTraversalTree(path []interface{}) *PathTreeNode {
+	if len(path) == 0 {
+		return &PathTreeNode{Operation: &Operation{OperationType: SelfReference}}
+	} else if len(path) == 1 {
+		return &PathTreeNode{Operation: &Operation{OperationType: TraversePath, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
+	}
+	return &PathTreeNode{
+		Operation: &Operation{OperationType: ShortPipe},
+		Lhs:       createTraversalTree(path[0:1]),
+		Rhs:       createTraversalTree(path[1:])}
+}

pkg/yqlib/operators_test.go

@@ -38,7 +38,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 	if s.document != "" {
 		inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml", 0)
 		if err != nil {
-			t.Error(err, s.document)
+			t.Error(err, s.document, s.expression)
 			return
 		}
 	} else {
@@ -55,7 +55,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 	results, err = treeNavigator.GetMatchingNodes(inputs, node)
 
 	if err != nil {
-		t.Error(err)
+		t.Error(fmt.Errorf("%v: %v", err, s.expression))
 		return
 	}
 	test.AssertResultComplexWithContext(t, s.expected, resultsToString(results), fmt.Sprintf("exp: %v\ndoc: %v", s.expression, s.document))
@@ -167,17 +167,17 @@ func documentScenarios(t *testing.T, title string, scenarios []expressionScenari
 			if s.document != "" {
 				node, err := treeCreator.ParsePath(s.expression)
 				if err != nil {
-					t.Error(err)
+					t.Error(err, s.expression)
 				}
 				err = streamEvaluator.Evaluate("sample.yaml", strings.NewReader(formattedDoc), node, printer)
 
 				if err != nil {
-					t.Error(err)
+					t.Error(err, s.expression)
 				}
 			} else {
 				err = streamEvaluator.EvaluateNew(s.expression, printer)
 				if err != nil {
-					t.Error(err)
+					t.Error(err, s.expression)
 				}
 			}
 

pkg/yqlib/path_parse_test.go

@@ -17,20 +17,35 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "[", "]"),
 		append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE"),
 	},
+	{
+		`.[]`,
+		append(make([]interface{}, 0), "TRAVERSE_ARRAY", "[", "]"),
+		append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
 	{
 		`.a[]`,
-		append(make([]interface{}, 0), "a", "SHORT_PIPE", "[]"),
+		append(make([]interface{}, 0), "a", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "]"),
-		append(make([]interface{}, 0), "a", "[]", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
 	},
 	{
 		`.a.[]`,
-		append(make([]interface{}, 0), "a", "SHORT_PIPE", "[]"),
+		append(make([]interface{}, 0), "a", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "]"),
-		append(make([]interface{}, 0), "a", "[]", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
+	},
+	{
+		`.a[0]`,
+		append(make([]interface{}, 0), "a", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
+		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
+	},
+	{
+		`.a.[0]`,
+		append(make([]interface{}, 0), "a", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
+		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
 	},
 	{
 		`.a[].c`,
-		append(make([]interface{}, 0), "a", "SHORT_PIPE", "[]", "SHORT_PIPE", "c"),
+		append(make([]interface{}, 0), "a", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "]", "SHORT_PIPE", "c"),
-		append(make([]interface{}, 0), "a", "[]", "SHORT_PIPE", "c", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "c", "SHORT_PIPE"),
 	},
 	{
 		`[3]`,
@@ -44,18 +59,18 @@ var pathTests = []struct {
 	},
 	{
 		`.a | .[].b == "apple"`,
-		append(make([]interface{}, 0), "a", "PIPE", "[]", "SHORT_PIPE", "b", "EQUALS", "apple (string)"),
+		append(make([]interface{}, 0), "a", "PIPE", "TRAVERSE_ARRAY", "[", "]", "SHORT_PIPE", "b", "EQUALS", "apple (string)"),
-		append(make([]interface{}, 0), "a", "[]", "b", "SHORT_PIPE", "apple (string)", "EQUALS", "PIPE"),
+		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "apple (string)", "EQUALS", "PIPE"),
 	},
 	{
 		`(.a | .[].b) == "apple"`,
-		append(make([]interface{}, 0), "(", "a", "PIPE", "[]", "SHORT_PIPE", "b", ")", "EQUALS", "apple (string)"),
+		append(make([]interface{}, 0), "(", "a", "PIPE", "TRAVERSE_ARRAY", "[", "]", "SHORT_PIPE", "b", ")", "EQUALS", "apple (string)"),
-		append(make([]interface{}, 0), "a", "[]", "b", "SHORT_PIPE", "PIPE", "apple (string)", "EQUALS"),
+		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "PIPE", "apple (string)", "EQUALS"),
 	},
 	{
 		`.[] | select(. == "*at")`,
-		append(make([]interface{}, 0), "[]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at (string)", ")"),
+		append(make([]interface{}, 0), "TRAVERSE_ARRAY", "[", "]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at (string)", ")"),
-		append(make([]interface{}, 0), "[]", "SELF", "*at (string)", "EQUALS", "SELECT", "PIPE"),
+		append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SELF", "*at (string)", "EQUALS", "SELECT", "PIPE"),
 	},
 	{
 		`[true]`,
@@ -89,8 +104,8 @@ var pathTests = []struct {
 	},
 	{
 		`{.a: .c, .b.[]: .f.g.[]}`,
-		append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "SHORT_PIPE", "[]", "CREATE_MAP", "f", "SHORT_PIPE", "g", "SHORT_PIPE", "[]", "}"),
+		append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "]", "CREATE_MAP", "f", "SHORT_PIPE", "g", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "]", "}"),
-		append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "[]", "SHORT_PIPE", "f", "g", "SHORT_PIPE", "[]", "SHORT_PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "f", "g", "SHORT_PIPE", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "SHORT_PIPE"),
 	},
 	{
 		`explode(.a.b)`,
@@ -122,7 +137,6 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "SELF", "ASSIGN_COMMENT", "str (string)"),
 		append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_COMMENT"),
 	},
-
 	{
 		`.a.b tag="!!str"`,
 		append(make([]interface{}, 0), "a", "SHORT_PIPE", "b", "ASSIGN_TAG", "!!str (string)"),

pkg/yqlib/path_tokeniser.go

@@ -22,7 +22,7 @@ const (
 	CloseCollect
 	OpenCollectObject
 	CloseCollectObject
-	SplatOrEmptyCollect
+	TraverseArrayCollect
 )
 
 type Token struct {
@@ -49,8 +49,9 @@ func (t *Token) toString() string {
 		return "{"
 	} else if t.TokenType == CloseCollectObject {
 		return "}"
-	} else if t.TokenType == SplatOrEmptyCollect {
+	} else if t.TokenType == TraverseArrayCollect {
-		return "[]?"
+		return ".["
+
 	} else {
 		return "NFI"
 	}
@@ -114,23 +115,6 @@ func unwrap(value string) string {
 	return value[1 : len(value)-1]
 }
 
-func arrayIndextoken(precedingDot bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		var numberString = string(m.Bytes)
-		startIndex := 1
-		if precedingDot {
-			startIndex = 2
-		}
-		numberString = numberString[startIndex : len(numberString)-1]
-		var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) // nolint
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-		op := &Operation{OperationType: TraversePath, Value: number, StringValue: numberString}
-		return &Token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil
-	}
-}
-
 func numberValue() lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		var numberString = string(m.Bytes)
@@ -188,7 +172,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`\(`), literalToken(OpenBracket, false))
 	lexer.Add([]byte(`\)`), literalToken(CloseBracket, true))
 
-	lexer.Add([]byte(`\.\[\]`), pathToken(false))
+	lexer.Add([]byte(`\.\[`), literalToken(TraverseArrayCollect, false))
 	lexer.Add([]byte(`\.\.`), opToken(RecursiveDescent))
 
 	lexer.Add([]byte(`,`), opToken(Union))
@@ -201,12 +185,15 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`or`), opToken(Or))
 	lexer.Add([]byte(`and`), opToken(And))
 	lexer.Add([]byte(`not`), opToken(Not))
+	lexer.Add([]byte(`\/\/`), opToken(Alternative))
 
 	lexer.Add([]byte(`documentIndex`), opToken(GetDocumentIndex))
 
 	lexer.Add([]byte(`style`), opAssignableToken(GetStyle, AssignStyle))
 
 	lexer.Add([]byte(`tag`), opAssignableToken(GetTag, AssignTag))
+	lexer.Add([]byte(`anchor`), opAssignableToken(GetAnchor, AssignAnchor))
+	lexer.Add([]byte(`alias`), opAssignableToken(GetAlias, AssignAlias))
 	lexer.Add([]byte(`filename`), opToken(GetFilename))
 	lexer.Add([]byte(`fileIndex`), opToken(GetFileIndex))
 	lexer.Add([]byte(`path`), opToken(GetPath))
@@ -228,8 +215,6 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte(`\s*\|=\s*`), opTokenWithPrefs(Assign, nil, &AssignOpPreferences{true}))
 
-	lexer.Add([]byte(`\.\[-?[0-9]+\]`), arrayIndextoken(true))
-
 	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
 
 	lexer.Add([]byte(`d[0-9]+`), documentToken())
@@ -249,9 +234,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`[Nn][Uu][Ll][Ll]`), nullValue())
 	lexer.Add([]byte(`~`), nullValue())
 
-	lexer.Add([]byte(`"[^ "]*"`), stringValue(true))
+	lexer.Add([]byte(`"[^"]*"`), stringValue(true))
-
-	lexer.Add([]byte(`\[\]`), literalToken(SplatOrEmptyCollect, true))
 
 	lexer.Add([]byte(`\[`), literalToken(OpenCollect, false))
 	lexer.Add([]byte(`\]`), literalToken(CloseCollect, true))
@@ -321,24 +304,16 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) {
 func (p *pathTokeniser) handleToken(tokens []*Token, index int, postProcessedTokens []*Token) (tokensAccum []*Token, skipNextToken bool) {
 	skipNextToken = false
 	token := tokens[index]
-	if token.TokenType == SplatOrEmptyCollect {
-		if index > 0 && tokens[index-1].TokenType == OperationToken &&
-			tokens[index-1].Operation.OperationType == TraversePath {
-			// must be a splat without a preceding dot , e.g. .a[]
-			// lets put a pipe in front of it, and convert it to a traverse "[]" token
-			pipeOp := &Operation{OperationType: ShortPipe, Value: "PIPE"}
 
-			postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: pipeOp})
+	if token.TokenType == TraverseArrayCollect {
+		//need to put a traverse array then a collect token
+		// do this by adding traverse then converting token to collect
 
-			traverseOp := &Operation{OperationType: TraversePath, Value: "[]", StringValue: "[]"}
+		op := &Operation{OperationType: TraverseArray, StringValue: "TRAVERSE_ARRAY"}
-			token = &Token{TokenType: OperationToken, Operation: traverseOp, CheckForPostTraverse: true}
+		postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
+
+		token = &Token{TokenType: OpenCollect}
 
-		} else {
-			// gotta be a collect empty array, we need to split this into two tokens
-			// one OpenCollect, the other CloseCollect
-			postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OpenCollect})
-			token = &Token{TokenType: CloseCollect, CheckForPostTraverse: true}
-		}
 	}
 
 	if index != len(tokens)-1 && token.AssignOperation != nil &&
@@ -356,5 +331,21 @@ func (p *pathTokeniser) handleToken(tokens []*Token, index int, postProcessedTok
 		op := &Operation{OperationType: ShortPipe, Value: "PIPE"}
 		postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
 	}
+	if index != len(tokens)-1 && token.CheckForPostTraverse &&
+		tokens[index+1].TokenType == OpenCollect {
+
+		op := &Operation{OperationType: ShortPipe, Value: "PIPE"}
+		postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
+
+		op = &Operation{OperationType: TraverseArray}
+		postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
+	}
+	if index != len(tokens)-1 && token.CheckForPostTraverse &&
+		tokens[index+1].TokenType == TraverseArrayCollect {
+
+		op := &Operation{OperationType: ShortPipe, Value: "PIPE"}
+		postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
+
+	}
 	return postProcessedTokens, skipNextToken
 }

release_instructions.txt

@@ -1,15 +1,9 @@
 - increment version in version.go
 - increment version in snapcraft.yaml
-- commit
-- tag git with same version number
 - make sure local build passes
-- push tag to git
+- tag git with same version number
-  - git push --tags
+- commit vX tag - this will trigger github actions
-- make local xcompile (builds binaries for all platforms)
+- use github actions to publish docker and make github release
-
-- git release
-  ./scripts/release.sh
-  ./scripts/upload.sh
 
 - snapcraft
   - will auto create a candidate, test it works then promote
@@ -30,7 +24,7 @@
 
 - docker
   - build and push latest and new version tag
-  - docker build .  -t mikefarah/yq:latest -t mikefarah/yq:VERSION 
+  - docker build .  -t mikefarah/yq:latest -t mikefarah/yq:3 -t mikefarah/yq:3.X
 
 - debian package
   - ensure you get all vendor dependencies before packaging

scripts/check.sh

@@ -3,7 +3,7 @@
 set -o errexit
 set -o pipefail
 
-./bin/golangci-lint run
+./bin/golangci-lint run --timeout=5m
 
 # ./bin/golangci-lint \
 #   --tests \

scripts/format.sh

@@ -2,3 +2,4 @@
 
 find . \( -path ./vendor \) -prune -o -name "*.go" -exec goimports -w {} \;
 go mod tidy
+go mod vendor

scripts/publish-docker.sh

@@ -6,4 +6,7 @@ docker build \
   --build-arg VERSION=${VERSION} \
   -t mikefarah/yq:latest \
   -t mikefarah/yq:${VERSION} \
+  -t mikefarah/yq:4 \
   .
+
+trivy image mikefarah/yq:${VERSION}

scripts/xcompile.sh

@@ -1,14 +1,24 @@
 #!/bin/bash
-
+set -e
 # This assumes that gonative and gox is installed as per the 'one time setup' instructions
 # at https://github.com/inconshreveable/gonative
 
 
-CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -output="build/yq_{{.OS}}_{{.Arch}}"
+CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -output="build/yq_{{.OS}}_{{.Arch}}" --osarch="darwin/amd64 freebsd/386 freebsd/amd64 freebsd/arm linux/386 linux/amd64 linux/arm linux/arm64 linux/mips linux/mips64 linux/mips64le linux/mipsle linux/ppc64 linux/ppc64le linux/s390x netbsd/386 netbsd/amd64 netbsd/arm openbsd/386 openbsd/amd64 windows/386 windows/amd64"
-# include non-default linux builds too
-CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -os=linux  -output="build/yq_{{.OS}}_{{.Arch}}"
 
 cd build
-rhash -r -a . -P -o checksums
+rhash -r -a . -o checksums
 
 rhash --list-hashes > checksums_hashes_order
+
+find . -executable -type f | xargs -I {} tar czvf {}.tar.gz {}
+
+# just in case find thinks this is executable...
+rm -f checksums_hashes_order.tar.gz 
+rm -f checksums.tar.gz
+
+rm yq_windows_386.exe.tar.gz
+rm yq_windows_amd64.exe.tar.gz
+
+zip yq_windows_386.zip yq_windows_386.exe
+zip yq_windows_amd64.zip yq_windows_amd64.exe

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.0.0-beta1'
+version: '4.1.0'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.

yq_test.go

@@ -1,60 +0,0 @@
-package main
-
-// import (
-// 	"fmt"
-// 	"runtime"
-// 	"testing"
-
-// 	"github.com/mikefarah/yq/v2/pkg/marshal"
-// 	"github.com/mikefarah/yq/v2/test"
-// )
-
-// func TestMultilineString(t *testing.T) {
-// 	testString := `
-// 	abcd
-// 	efg`
-// 	formattedResult, _ := marshal.NewYamlConverter().YamlToString(testString, false)
-// 	test.AssertResult(t, testString, formattedResult)
-// }
-
-// func TestNewYaml(t *testing.T) {
-// 	result, _ := newYaml([]string{"b.c", "3"})
-// 	formattedResult := fmt.Sprintf("%v", result)
-// 	test.AssertResult(t,
-// 		"[{b [{c 3}]}]",
-// 		formattedResult)
-// }
-
-// func TestNewYamlArray(t *testing.T) {
-// 	result, _ := newYaml([]string{"[0].cat", "meow"})
-// 	formattedResult := fmt.Sprintf("%v", result)
-// 	test.AssertResult(t,
-// 		"[[{cat meow}]]",
-// 		formattedResult)
-// }
-
-// func TestNewYaml_WithScript(t *testing.T) {
-// 	writeScript = "examples/instruction_sample.yaml"
-// 	expectedResult := `b:
-//   c: cat
-//   e:
-//   - name: Mike Farah`
-// 	result, _ := newYaml([]string{""})
-// 	actualResult, _ := marshal.NewYamlConverter().YamlToString(result, true)
-// 	test.AssertResult(t, expectedResult, actualResult)
-// }
-
-// func TestNewYaml_WithUnknownScript(t *testing.T) {
-// 	writeScript = "fake-unknown"
-// 	_, err := newYaml([]string{""})
-// 	if err == nil {
-// 		t.Error("Expected error due to unknown file")
-// 	}
-// 	var expectedOutput string
-// 	if runtime.GOOS == "windows" {
-// 		expectedOutput = `open fake-unknown: The system cannot find the file specified.`
-// 	} else {
-// 		expectedOutput = `open fake-unknown: no such file or directory`
-// 	}
-// 	test.AssertResult(t, expectedOutput, err.Error())
-// }