1
0
mirror of https://github.com/taigrr/yq synced 2025-01-18 04:53:17 -08:00

Compare commits

..

101 Commits

Author SHA1 Message Date
Mike Farah
f42355cf9f Force re-release 2021-02-10 17:12:54 +11:00
Mike Farah
44bc9971c1 Fixed write-inplace with no expression 2021-02-10 17:06:16 +11:00
Ben Moss
54cf6cf282 Fix pretty-printing
Fixes #716 #715
2021-02-10 16:56:44 +11:00
Mike Farah
29db5ebf4d Version increment 2021-02-10 16:55:05 +11:00
Mike Farah
abaf299115 Preserve comments on map keys 2021-02-10 16:48:10 +11:00
Mike Farah
e9c5573a89 Added space example to docs 2021-02-05 15:00:46 +11:00
Mike Farah
e0fdbe7f35 update deps 2021-02-05 14:57:45 +11:00
Mike Farah
ebfc7212e2 Fixed merge dropping anchors 2021-02-05 14:40:16 +11:00
Mike Farah
ca10ede102 improving docs 2021-02-05 13:32:07 +11:00
Mike Farah
e76a2f3de6 Fixing special character example 2021-02-05 13:29:35 +11:00
Mike Farah
b702686891 Added not equals operator 2021-02-05 09:54:03 +11:00
Mike Farah
c284d4c6b1 Equals now only compares scalars 2021-02-05 09:49:40 +11:00
Mike Farah
63883d3000 Fixed delete bug 2021-02-04 13:47:59 +11:00
Mike Farah
3014192ce8 fixed instructions 2021-02-04 13:12:16 +11:00
Mike Farah
ebeb1146ba v4.5.0 2021-02-04 13:04:53 +11:00
Mike Farah
0bb53ec770 Fixed delete bug 2021-02-04 12:48:07 +11:00
Mike Farah
a50e154652 Added variable doc 2021-02-04 12:44:03 +11:00
Mike Farah
f91d30e46c Fixed variable precedence 2021-02-04 12:39:04 +11:00
Mike Farah
07b053f040 Fixing op precedences 2021-02-04 12:18:54 +11:00
Mike Farah
35c97e832e Fixing op precedences 2021-02-03 17:11:47 +11:00
Mike Farah
8a5c47906e Added variables 2021-02-03 15:51:26 +11:00
Mike Farah
a366cacad3 Dont create entries when selecting 2021-02-03 11:56:35 +11:00
Mike Farah
7c8d3b9e70 Pass context through operators
Allows more sophisticated functionality
2021-02-03 11:56:35 +11:00
Mike Farah
ca10642e23 Added funding button 2021-02-01 15:50:57 +11:00
zy
73518f3915 change version from master to latest(v4) 2021-01-27 10:09:32 +11:00
zy
348ddb7a59 fix: go install fails
Installation by go will result in an error:
module declares its path as: xxx,
but was required as: xxx.

Go install with master branch will fix it.

fix: #676
2021-01-27 10:09:32 +11:00
Mike Farah
f46fe384bd Fixed length of null to be zero 2021-01-26 10:21:16 +11:00
Mike Farah
071ec3c08c fixing docker pipeline for nextime 2021-01-21 21:15:21 +11:00
Mike Farah
c2ed3a3e6d attempt to fix dockre 2021-01-21 21:11:04 +11:00
Mike Farah
cc4c69bc89 Fixed bad docker version 2021-01-21 21:07:55 +11:00
Mike Farah
917fd0eb6a Bump version 2021-01-18 14:10:35 +11:00
Mike Farah
b2056a2056 Can add and merge append to null 2021-01-18 13:58:46 +11:00
evnp
ba59201217 Don't escape HTML chars when converting to json
json.Encoder and json.Marshal implicitly use HTMLEscape to convert
>, <, &, with \u003c, \u003e, \u0026. This behavior carries over
to yq, where chars will be escaped when outputting json but not when
outputting yaml.

This changeset disables this behavior via encoder.SetEscapeHTML(false).
Unfortunately there is no equivalent option for json.Marshal, so its
single usage has been replaced with an encoder (with escaping disabled).
2021-01-18 13:33:02 +11:00
Mike Farah
cfddee9101 Fixed cross-function combinatorial bug 2021-01-18 13:28:40 +11:00
Mike Farah
1941bb66a5 wip 2021-01-18 10:15:31 +11:00
Mike Farah
29af9e4c63 thoughts 2021-01-16 14:56:52 +11:00
Mike Farah
af3a9ae846 cross function fix wip 2021-01-16 14:09:49 +11:00
Mike Farah
5fb88627ca Fixed doker instructions 2021-01-15 09:44:02 +11:00
Mike Farah
4fc3793fcb Fixing multiply doc 2021-01-14 20:28:57 +11:00
Mike Farah
d612897e89 Incrementing version 2021-01-14 19:50:37 +11:00
Mike Farah
806f906041 Added keys operator 2021-01-14 15:45:07 +11:00
Mike Farah
e8b2f6e383 Added split string operator 2021-01-14 15:05:50 +11:00
Mike Farah
e419b5a39d Added join strings operator 2021-01-14 14:46:50 +11:00
Mike Farah
a5b6e08282 Split doc operator 2021-01-14 14:25:31 +11:00
Mike Farah
2417c0ee8f Fixing add,multiply,alternative operator precendences 2021-01-14 11:16:04 +11:00
Mike Farah
d3b2e37b66 Fixed remove comments example 2021-01-14 09:12:14 +11:00
Mike Farah
54723c1a36 Dont use pointer for env prefs (avoid nil) 2021-01-13 17:00:53 +11:00
Mike Farah
0318c80d33 Dont use pointer for recursive prefs (avoid nil) 2021-01-13 17:00:03 +11:00
Mike Farah
8980846632 Dont use pointer for multiply prefs (avoid nil) 2021-01-13 16:59:01 +11:00
Mike Farah
0fb62d3ae1 Dont use pointer for commment prefs (avoid nil) 2021-01-13 16:56:24 +11:00
Mike Farah
85398727ad Added merge if empty 2021-01-13 16:54:28 +11:00
Mike Farah
0bf3d781f6 Added operator level doc 2021-01-13 15:23:26 +11:00
Mike Farah
5c15936bf3 Incrementing version 2021-01-13 10:29:06 +11:00
Mike Farah
b0735d8152 Renaming pathtree to expression 2021-01-13 10:25:26 +11:00
Mike Farah
f17cbfd007 Removed global vars 2021-01-13 10:04:52 +11:00
Mike Farah
7b7ab70286 UnwrapDoc now private 2021-01-13 10:00:51 +11:00
Mike Farah
43165fa340 Moved eval function to eval interface 2021-01-13 09:35:57 +11:00
Mikhail Katychev
feda9f044d added lib_test.go 2021-01-13 09:22:40 +11:00
Mikhail Katychev
85629af59c added EvaluateNodes and EvaluateCandidateNodes to yqlib 2021-01-13 09:22:40 +11:00
Mike Farah
5fc26e9453 Merge now copies anchor names 2021-01-13 09:21:16 +11:00
Mike Farah
07a6fa4df5 Fixed creation of candidateNode in operators to include file metadata 2021-01-12 19:36:28 +11:00
Mike Farah
ec29ee7c14 Cleaning up exposed public api 2021-01-12 09:58:50 +11:00
Mike Farah
24538c0cc7 Fixed tag operator for top level node 2021-01-12 09:45:57 +11:00
Mike Farah
61398bfd3a Fixed equals operator for top level node 2021-01-12 09:40:37 +11:00
Mike Farah
f7d95021c1 Fixed has operator for top level node 2021-01-12 09:30:24 +11:00
Mike Farah
6bb8b1fb77 fixing exposed functions and interfaces 2021-01-11 17:13:48 +11:00
Mike Farah
5e2c19cc86 fixing exposed functions and interfaces 2021-01-11 16:46:28 +11:00
Mike Farah
30c269a66c Better add documentation 2021-01-11 15:52:06 +11:00
Mike Farah
70a1c60d7b Added scalar addition 2021-01-11 15:43:50 +11:00
Mike Farah
e4d48bbc0d Fixed collect at document level 2021-01-11 14:44:53 +11:00
Mike Farah
369ff94ad0 Better error handling will empty env 2021-01-11 14:38:53 +11:00
Mike Farah
1c7fb14631 Better recursive decent docs 2021-01-10 11:27:18 +11:00
Mike Farah
f4de5c4300 Better docIndex docs 2021-01-10 11:18:13 +11:00
Mike Farah
a72c14f06b Better env docs 2021-01-10 11:09:59 +11:00
Mike Farah
5ed52aca66 Merged env commands in :eye-roll: 2021-01-10 10:51:23 +11:00
Mike Farah
5a5ac0dfef Merge branch 'env_var' 2021-01-10 10:50:31 +11:00
Mike Farah
5a55869745 Bump version 2021-01-09 12:23:20 +11:00
Mike Farah
15e18bb98b Error when passing files and using null-input flag 2021-01-09 12:23:06 +11:00
Mike Farah
644063646e Env Ops! 2021-01-09 12:06:19 +11:00
Mike Farah
aabed1a237 strenv 2021-01-09 11:38:08 +11:00
Mike Farah
c12764dba8 wip 2021-01-08 21:09:43 +11:00
Mike Farah
1d5ecb244d wip 2021-01-08 21:07:46 +11:00
Mike Farah
7798a141cf Bumped go yaml for comment hanlding fixes 2021-01-08 20:56:54 +11:00
Mike Farah
b7583a538d Added webi 2021-01-08 12:14:36 +11:00
Mike Farah
529e88b630 Added recurse examples 2021-01-08 12:11:29 +11:00
Mike Farah
658006eb93 Added another delete example 2021-01-08 11:59:49 +11:00
Mike Farah
2743a7e058 Can assign-update tag 2021-01-06 20:44:28 +11:00
Mike Farah
f95862fba3 Can assign-update style 2021-01-06 20:37:53 +11:00
Mike Farah
3f58c4bc38 Can assign-update aliases and anchors 2021-01-06 20:30:48 +11:00
Mike Farah
a7975df7cd Can assign-update comments 2021-01-06 20:22:50 +11:00
Mike Farah
3d5a5e0360 updating readme 2021-01-06 15:33:20 +11:00
Mike Farah
155755ae2f brew v3! 2021-01-06 15:22:29 +11:00
Mike Farah
601c13531c Updated collect objcet doc 2021-01-06 15:20:54 +11:00
Mike Farah
69d00c89df Added shorthand document index selection 2021-01-05 13:28:37 +11:00
Mike Farah
2faff7b05f Unwrap node in get tag to return proper tag at root level 2021-01-05 13:23:03 +11:00
Mike Farah
165949041d Added v3 snap instructions 2021-01-05 13:05:03 +11:00
Mike Farah
dbd7ab0f13 Refactored doc generation, add fi fileIndex alias 2021-01-02 10:49:33 +11:00
Mike Farah
6d512ad718 Fixed updating yaml from other files 2021-01-02 10:27:32 +11:00
Mike Farah
4fef4a7ab1 update issue template, instruct questions to be raised in disussion 2020-12-31 13:43:44 +11:00
Mike Farah
dcb17b51a9 fixed heading 2020-12-31 09:33:15 +11:00
Mike Farah
385417556d added tar.gz instructions 2020-12-31 09:30:59 +11:00
62 changed files with 216 additions and 1105 deletions

View File

@@ -1 +1 @@
bin/*
bin

View File

@@ -6,5 +6,26 @@ RUN set -e -x \
&& /opt/devtools.sh
ENV PATH=/go/bin:$PATH
# install mkdocs
RUN set -ex \
&& buildDeps=' \
build-essential \
python3-dev \
' \
&& apt-get update && apt-get install -y --no-install-recommends \
$buildDeps \
python3 \
python3-setuptools \
python3-wheel \
python3-pip \
&& pip3 install --upgrade \
pip \
'Markdown>=2.6.9' \
'mkdocs>=0.16.3' \
'mkdocs-material>=1.10.1' \
'markdown-include>=0.5.1' \
&& apt-get purge -y --auto-remove $buildDeps \
&& rm -rf /var/lib/apt/lists/*
ENV CGO_ENABLED 0
ENV GOPATH /go:/yq

View File

@@ -17,7 +17,6 @@ help:
@echo ' make vendor Install dependencies to vendor directory.'
@echo ' make format Run code formatter.'
@echo ' make check Run static code analysis (lint).'
@echo ' make secure Run gosec.'
@echo ' make test Run tests on project.'
@echo ' make cover Run tests and capture code coverage metrics on project.'
@echo ' make clean Clean the directory tree of produced artifacts.'
@@ -85,10 +84,6 @@ format: vendor
check: format
${DOCKRUN} bash ./scripts/check.sh
.PHONY: secure
secure:
${DOCKRUN} bash ./scripts/secure.sh
.PHONY: test
test: check
${DOCKRUN} bash ./scripts/test.sh
@@ -101,6 +96,11 @@ cover: check
@find cover -type d -exec chmod 755 {} \; || :
@find cover -type f -exec chmod 644 {} \; || :
.PHONY: build-docs
build-docs: prepare mkdocs.yml mkdocs/*
${DOCKRUN} mkdocs build
@find docs -type d -exec chmod 755 {} \; || :
@find docs -type f -exec chmod 644 {} \; || :
.PHONY: release
release: xcompile

View File

@@ -119,8 +119,6 @@ See [webi](https://webinstall.dev/)
Supported by @adithyasunil26 (https://github.com/webinstall/webi-installers/tree/master/yq)
### Windows:
[![Chocolatey](https://img.shields.io/chocolatey/v/yq.svg)](https://chocolatey.org/packages/yq)
[![Chocolatey](https://img.shields.io/chocolatey/dt/yq.svg)](https://chocolatey.org/packages/yq)
```
choco install yq
```
@@ -153,21 +151,19 @@ sudo apt install yq -y
Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
## Features
- [Detailed documentation with many examples](https://mikefarah.gitbook.io/yq/)
- Written in portable go, so you can download a lovely dependency free binary
- Uses similar syntax as `jq` but works with YAML and JSON files
- Fully supports multi document yaml files
- Colorized yaml output
- [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/operators/traverse-read)
- [Sort yaml by keys](https://mikefarah.gitbook.io/yq/operators/sort-keys)
- Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/operators/comment-operators), [styling](https://mikefarah.gitbook.io/yq/operators/style), [tags](https://mikefarah.gitbook.io/yq/operators/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators).
- [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/v/v4.x/traverse)
- [Sort yaml by keys](https://mikefarah.gitbook.io/yq/v/v4.x/sort-keys)
- Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/comment-operators), [styling](https://mikefarah.gitbook.io/yq/style), [tags](https://mikefarah.gitbook.io/yq/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/anchor-and-alias-operators).
- [Update yaml inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
- [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map)
- [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/v/v4.x/select#select-and-update-matching-values-in-map)
- Keeps yaml formatting and comments when updating (though there are issues with whitespace)
- [Convert to/from json to yaml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
- [Pipe data in by using '-'](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate)
- [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
- [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things.
## [Usage](https://mikefarah.gitbook.io/yq/)

View File

@@ -17,9 +17,6 @@ func createEvaluateAllCommand() *cobra.Command {
Example: `
# merges f2.yml into f1.yml (inplace)
yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml
# use '-' as a filename to read from STDIN
cat file2.yml | yq ea '.a.b' file1.yml - file3.yml
`,
Long: "Evaluate All:\nUseful when you need to run an expression across several yaml documents or files. Consumes more memory than eval",
RunE: evaluateAll,

View File

@@ -21,9 +21,6 @@ yq e '.a.b | length' f1.yml f2.yml
# prints out the file
yq e sample.yaml
# use '-' as a filename to read from STDIN
cat file2.yml | yq e '.a.b' file1.yml - file3.yml
# prints a new yaml document
yq e -n '.a.b.c = "cat"'

View File

@@ -11,7 +11,7 @@ var (
GitDescribe string
// Version is main version number that is being run at the moment.
Version = "4.7.1"
Version = "4.5.1"
// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release

View File

@@ -1,4 +1,4 @@
FROM mikefarah/yq:4.7.1
FROM mikefarah/yq:4.5.1
COPY entrypoint.sh /entrypoint.sh

8
go.mod
View File

@@ -3,12 +3,12 @@ module github.com/mikefarah/yq/v4
require (
github.com/elliotchance/orderedmap v1.4.0
github.com/fatih/color v1.10.0
github.com/goccy/go-yaml v1.8.9
github.com/jinzhu/copier v0.2.8
github.com/spf13/cobra v1.1.3
github.com/goccy/go-yaml v1.8.8
github.com/jinzhu/copier v0.2.3
github.com/spf13/cobra v1.1.1
github.com/timtadh/data-structures v0.5.3 // indirect
github.com/timtadh/lexmachine v0.2.2
golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e // indirect
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)

26
go.sum
View File

@@ -52,8 +52,8 @@ github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTM
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/goccy/go-yaml v1.8.9 h1:4AEXg2qx+/w29jXnXpMY6mTckmYu1TMoHteKuMf0HFg=
github.com/goccy/go-yaml v1.8.9/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
github.com/goccy/go-yaml v1.8.8 h1:MGfRB1GeSn/hWXYWS2Pt67iC2GJNnebdIro01ddyucA=
github.com/goccy/go-yaml v1.8.8/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -101,8 +101,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jinzhu/copier v0.2.8 h1:N8MbL5niMwE3P4dOwurJixz5rMkKfujmMRFmAanSzWE=
github.com/jinzhu/copier v0.2.8/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
github.com/jinzhu/copier v0.2.3 h1:Oe09ju+9qft7TffZ7l/04AB2f8u1+V4ZMxmp/nnqeOs=
github.com/jinzhu/copier v0.2.3/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -168,9 +168,10 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@@ -179,6 +180,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -255,9 +257,10 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e h1:XNp2Flc/1eWQGk5BLzqTAN7fQIwIbfyVTuVxXxZh73M=
golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -267,6 +270,7 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -279,6 +283,7 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
@@ -305,6 +310,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -315,9 +321,11 @@ gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -8,10 +8,9 @@ import (
)
type CandidateNode struct {
Node *yaml.Node // the actual node
Parent *CandidateNode // parent node
Path []interface{} /// the path we took to get to this node
Document uint // the document index of this node
Node *yaml.Node // the actual node
Path []interface{} /// the path we took to get to this node
Document uint // the document index of this node
Filename string
FileIndex int
// when performing op against all nodes given, this will treat all the nodes as one
@@ -32,7 +31,6 @@ func (n *CandidateNode) CreateChild(path interface{}, node *yaml.Node) *Candidat
return &CandidateNode{
Node: node,
Path: n.createChildPath(path),
Parent: n,
Document: n.Document,
Filename: n.Filename,
FileIndex: n.FileIndex,
@@ -88,14 +86,7 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) {
if n.Node.Style == 0 {
n.Node.Style = other.Node.Style
}
if other.Node.FootComment != "" {
n.Node.FootComment = other.Node.FootComment
}
if other.Node.HeadComment != "" {
n.Node.HeadComment = other.Node.HeadComment
}
if other.Node.LineComment != "" {
n.Node.LineComment = other.Node.LineComment
}
n.Node.FootComment = n.Node.FootComment + other.Node.FootComment
n.Node.HeadComment = n.Node.HeadComment + other.Node.HeadComment
n.Node.LineComment = n.Node.LineComment + other.Node.LineComment
}

View File

@@ -101,7 +101,7 @@ will output
- hello
```
## Append to array
## Update array (append)
Given a sample.yml file of:
```yaml
a:
@@ -127,36 +127,6 @@ b:
- 4
```
## Relative append
Given a sample.yml file of:
```yaml
a:
a1:
b:
- cat
a2:
b:
- dog
a3: {}
```
then
```bash
yq eval '.a[].b += ["mouse"]' sample.yml
```
will output
```yaml
a:
a1:
b:
- cat
- mouse
a2:
b:
- dog
- mouse
a3: {b: [mouse]}
```
## String concatenation
Given a sample.yml file of:
```yaml
@@ -173,6 +143,22 @@ a: catmeow
b: meow
```
## Relative string concatenation
Given a sample.yml file of:
```yaml
a: cat
b: meow
```
then
```bash
yq eval '.a += .b' sample.yml
```
will output
```yaml
a: catmeow
b: meow
```
## Number addition - float
If the lhs or rhs are floats then the expression will be calculated with floats.
@@ -209,20 +195,18 @@ a: 7
b: 4
```
## Increment numbers
## Increment number
Given a sample.yml file of:
```yaml
a: 3
b: 5
```
then
```bash
yq eval '.[] += 1' sample.yml
yq eval '.a += 1' sample.yml
```
will output
```yaml
a: 4
b: 6
```
## Add to null

View File

@@ -1,4 +1,4 @@
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference aliases and remove anchor names).
`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.

View File

@@ -1,4 +1,10 @@
This operator is used to update node values. It can be used in either the:
### plain form: `=`
Which will assign the LHS node values to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.
### relative form: `|=`
This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment.
## Create yaml file
Running
```bash

View File

@@ -1,15 +1,4 @@
Use these comment operators to set or retrieve comments.
Like the `=` and `|=` assign operators, the same syntax applies when updating comments:
### plain form: `=`
This will assign the LHS nodes comments to the expression on the RHS. The RHS is run against the matching nodes in the pipeline
### relative form: `|=`
Similar to the plain form, however the RHS evaluates against each matching LHS node! This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.
## Set line comment
Given a sample.yml file of:
```yaml

View File

@@ -35,26 +35,6 @@ will output
0
```
## Get file indices of multiple documents
Given a sample.yml file of:
```yaml
a: cat
```
And another sample another.yml file of:
```yaml
a: cat
```
then
```bash
yq eval-all 'fileIndex' sample.yml another.yml
```
will output
```yaml
0
---
1
```
## Get file index alias
Given a sample.yml file of:
```yaml

View File

@@ -19,29 +19,6 @@ true
false
```
## Select, checking for existence of deep paths
Simply pipe in parent expressions into `has`
Given a sample.yml file of:
```yaml
- a:
b:
c: cat
- a:
b:
d: dog
```
then
```bash
yq eval '.[] | select(.a.b | has("c"))' sample.yml
```
will output
```yaml
a:
b:
c: cat
```
## Has array index
Given a sample.yml file of:
```yaml

View File

@@ -1,34 +1,19 @@
Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.
Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
## Objects and arrays - merging
Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged.
To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?.
Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
### Merge Flags
You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`. See examples below
Multiplication of strings and numbers are not yet supported.
- `+` to append arrays
- `?` to only merge existing fields
- `d` to deeply merge arrays
### Merging files
## Merging files
Note the use of `eval-all` to ensure all documents are loaded into memory.
```bash
yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml
```
## Multiply integers
Running
```bash
yq eval --null-input '3 * 4'
```
will output
```yaml
12
```
## Merge objects together, returning merged result only
Given a sample.yml file of:
```yaml
@@ -205,32 +190,6 @@ thing:
- 4
```
## Merge, deeply merging arrays
Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.
Given a sample.yml file of:
```yaml
a:
- name: fred
age: 12
- name: bob
age: 32
b:
- name: fred
age: 34
```
then
```bash
yq eval '.a *d .b' sample.yml
```
will output
```yaml
- name: fred
age: 34
- name: bob
age: 32
```
## Merge to prefix an element
Given a sample.yml file of:
```yaml

View File

@@ -1,76 +0,0 @@
Reduce is a powerful way to process a collection of data into a new form.
```
<exp> as $<name> ireduce (<init>; <block>)
```
e.g.
```
.[] as $item ireduce (0; . + $item)
```
On the LHS we are configuring the collection of items that will be reduced `<exp>` as well as what each element will be called `$<name>`. Note that the array has been splatted into its individual elements.
On the RHS there is `<init>`, the starting value of the accumulator and `<block>`, the expression that will update the accumulator for each element in the collection. Note that within the block expression, `.` will evaluate to the current value of the accumulator.
## yq vs jq syntax
Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`).
To that end, the reduce operator is called `ireduce` for backwards compatability if a `jq` like prefix version of `reduce` is ever added.
## Sum numbers
Given a sample.yml file of:
```yaml
- 10
- 2
- 5
- 3
```
then
```bash
yq eval '.[] as $item ireduce (0; . + $item)' sample.yml
```
will output
```yaml
20
```
## Merge all yaml files together
Given a sample.yml file of:
```yaml
a: cat
```
And another sample another.yml file of:
```yaml
b: dog
```
then
```bash
yq eval-all '. as $item ireduce ({}; . * $item )' sample.yml another.yml
```
will output
```yaml
a: cat
b: dog
```
## Convert an array to an object
Given a sample.yml file of:
```yaml
- name: Cathy
has: apples
- name: Bob
has: bananas
```
then
```bash
yq eval '.[] as $item ireduce ({}; .[$item | .name] = ($item | .has) )' sample.yml
```
will output
```yaml
Cathy: apples
Bob: bananas
```

View File

@@ -18,42 +18,6 @@ will output
cat; meow; 1; ; true
```
## Substitute / Replace string
This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
Note the use of `|=` to run in context of the current string value.
Given a sample.yml file of:
```yaml
a: dogs are great
```
then
```bash
yq eval '.a |= sub("dogs", "cats")' sample.yml
```
will output
```yaml
a: cats are great
```
## Substitute / Replace string with regex
This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
Note the use of `|=` to run in context of the current string value.
Given a sample.yml file of:
```yaml
a: cat
b: heat
```
then
```bash
yq eval '.[] |= sub("(a)", "${1}r")' sample.yml
```
will output
```yaml
a: cart
b: heart
```
## Split strings
Given a sample.yml file of:
```yaml

View File

@@ -1,71 +0,0 @@
## Number subtraction - float
If the lhs or rhs are floats then the expression will be calculated with floats.
Given a sample.yml file of:
```yaml
a: 3
b: 4.5
```
then
```bash
yq eval '.a = .a - .b' sample.yml
```
will output
```yaml
a: -1.5
b: 4.5
```
## Number subtraction - float
If the lhs or rhs are floats then the expression will be calculated with floats.
Given a sample.yml file of:
```yaml
a: 3
b: 4.5
```
then
```bash
yq eval '.a = .a - .b' sample.yml
```
will output
```yaml
a: -1.5
b: 4.5
```
## Number subtraction - int
If both the lhs and rhs are ints then the expression will be calculated with ints.
Given a sample.yml file of:
```yaml
a: 3
b: 4
```
then
```bash
yq eval '.a = .a - .b' sample.yml
```
will output
```yaml
a: -1
b: 4
```
## Decrement numbers
Given a sample.yml file of:
```yaml
a: 3
b: 5
```
then
```bash
yq eval '.[] -= 1' sample.yml
```
will output
```yaml
a: 2
b: 4
```

View File

@@ -1,44 +1,10 @@
# Operators
In `yq` expressions are made up of operators and pipes. A context of nodes is passed through the expression and each operation takes the context as input and returns a new context as output. That output is piped in as input for the next operation in the expression. To begin with, the context is set to the first yaml document of the first yaml file (if processing in sequence using eval).
In `yq` expressions are made up of operators. Operators have 0-2 arguments and run against the current 'matching' nodes in the expression tree.
Lets look at a couple of examples.
## Example 1 - simple example
Given a document like:
```yaml
- [a]
- "cat"
```
with an expression:
```
.[] | length
```
`yq` will initially set the context as single node of the entire yaml document, an array of two elements.
```yaml
- [a]
- "cat"
```
This gets piped into the splat operator `.[]` which will split out the context into a collection of two nodes `[a]` and `"cat"`. Note that this is _not_ a yaml array.
The `length` operator take no arguments, and will simply return the length of _each_ matching node in the context. So for the context of `[a]` and `"cat"`, it will return a new context of `1` and `3`.
This being the last operation in the expression, the results will be printed out:
```
1
3
```
# Example 2 - operators with arguments.
The `length` operator take no arguments, and will simply return the length of _each_ matching node. So if there were 2 nodes, one string and one array, length will update the 'matching' nodes context to be two new numeric scalar nodes representing the lengths of the orignal 'matching' nodes.
The `=` operator takes two arguments, a `lhs` expression and `rhs` expression. It runs the 'matching' nodes context against the `lhs` expression to find the nodes to update, lets call it `lhsNodes`, and then runs the matching nodes against the `rhs` to find the new values, lets call that `rhsNodes`. It updates the `lhsNodes` values with the `rhsNodes` values and _returns the original matching nodes_. This is important, where length changed the matching nodes to be new nodes with the length values, `=` returns the original matching nodes, albeit with some of the nodes values updated. So `.a = 3` will still return the parent matching node, but with the matching child updated.

View File

@@ -1,4 +1,4 @@
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference aliases and remove anchor names).
`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.

View File

@@ -1,11 +1 @@
Use these comment operators to set or retrieve comments.
Like the `=` and `|=` assign operators, the same syntax applies when updating comments:
### plain form: `=`
This will assign the LHS nodes comments to the expression on the RHS. The RHS is run against the matching nodes in the pipeline
### relative form: `|=`
Similar to the plain form, however the RHS evaluates against each matching LHS node! This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.

View File

@@ -1,20 +0,0 @@
Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.
## Objects and arrays - merging
Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged.
Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
### Merge Flags
You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`. See examples below
- `+` to append arrays
- `?` to only merge existing fields
- `d` to deeply merge arrays
### Merging files
Note the use of `eval-all` to ensure all documents are loaded into memory.
```bash
yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml
```

View File

@@ -0,0 +1,15 @@
Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?.
Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
Multiplication of strings and numbers are not yet supported.
## Merging files
Note the use of `eval-all` to ensure all documents are loaded into memory.
```bash
yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml
```

View File

@@ -1,21 +0,0 @@
Reduce is a powerful way to process a collection of data into a new form.
```
<exp> as $<name> ireduce (<init>; <block>)
```
e.g.
```
.[] as $item ireduce (0; . + $item)
```
On the LHS we are configuring the collection of items that will be reduced `<exp>` as well as what each element will be called `$<name>`. Note that the array has been splatted into its individual elements.
On the RHS there is `<init>`, the starting value of the accumulator and `<block>`, the expression that will update the accumulator for each element in the collection. Note that within the block expression, `.` will evaluate to the current value of the accumulator.
## yq vs jq syntax
Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`).
To that end, the reduce operator is called `ireduce` for backwards compatability if a `jq` like prefix version of `reduce` is ever added.

View File

@@ -5,6 +5,9 @@ import (
"strings"
)
var myPathTokeniser = newExpressionTokeniser()
var myPathPostfixer = newExpressionPostFixer()
type ExpressionNode struct {
Operation *Operation
Lhs *ExpressionNode
@@ -16,21 +19,19 @@ type ExpressionParser interface {
}
type expressionParserImpl struct {
pathTokeniser expressionTokeniser
pathPostFixer expressionPostFixer
}
func NewExpressionParser() ExpressionParser {
return &expressionParserImpl{newExpressionTokeniser(), newExpressionPostFixer()}
return &expressionParserImpl{}
}
func (p *expressionParserImpl) ParseExpression(expression string) (*ExpressionNode, error) {
tokens, err := p.pathTokeniser.Tokenise(expression)
tokens, err := myPathTokeniser.Tokenise(expression)
if err != nil {
return nil, err
}
var Operations []*Operation
Operations, err = p.pathPostFixer.ConvertToPostfix(tokens)
Operations, err = myPathPostfixer.ConvertToPostfix(tokens)
if err != nil {
return nil, err
}

View File

@@ -12,26 +12,6 @@ var pathTests = []struct {
expectedTokens []interface{}
expectedPostFix []interface{}
}{
{
`[]|join(".")`,
append(make([]interface{}, 0), "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")"),
append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE", ". (string)", "JOIN", "PIPE"),
},
{
`{"cool": .b or .c}`,
append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "b", "OR", "c", "}"),
append(make([]interface{}, 0), "cool (string)", "b", "c", "OR", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
},
{
`{"cool": []|join(".")}`,
append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")", "}"),
append(make([]interface{}, 0), "cool (string)", "EMPTY", "COLLECT", "SHORT_PIPE", ". (string)", "JOIN", "PIPE", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
},
{
`.a as $item ireduce (0; . + $item)`, // note - add code to shuffle reduce to this position for postfix
append(make([]interface{}, 0), "a", "ASSIGN_VARIABLE", "GET_VARIABLE", "REDUCE", "(", "0 (int64)", "BLOCK", "SELF", "ADD", "GET_VARIABLE", ")"),
append(make([]interface{}, 0), "a", "GET_VARIABLE", "ASSIGN_VARIABLE", "0 (int64)", "SELF", "GET_VARIABLE", "ADD", "BLOCK", "REDUCE"),
},
{
`.a | .b | .c`,
append(make([]interface{}, 0), "a", "PIPE", "b", "PIPE", "c"),

View File

@@ -100,9 +100,6 @@ func multiplyWithPrefs() lex.Action {
if strings.Contains(options, "?") {
prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
}
if strings.Contains(options, "d") {
prefs.DeepMergeArrays = true
}
op := &Operation{OperationType: multiplyOpType, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs}
return &token{TokenType: operationToken, Operation: op}, nil
}
@@ -189,7 +186,7 @@ func getVariableOpToken() lex.Action {
return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
value := string(m.Bytes)
value = value[1:]
value = value[1 : len(value)-1]
getVarOperation := createValueOperation(value, value)
getVarOperation.OperationType = getVariableOpType
@@ -255,8 +252,6 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`or`), opToken(orOpType))
lexer.Add([]byte(`and`), opToken(andOpType))
lexer.Add([]byte(`not`), opToken(notOpType))
lexer.Add([]byte(`ireduce`), opToken(reduceOpType))
lexer.Add([]byte(`;`), opToken(blockOpType))
lexer.Add([]byte(`\/\/`), opToken(alternativeOpType))
lexer.Add([]byte(`documentIndex`), opToken(getDocumentIndexOpType))
@@ -264,8 +259,6 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`splitDoc`), opToken(splitDocumentOpType))
lexer.Add([]byte(`join`), opToken(joinStringOpType))
lexer.Add([]byte(`sub`), opToken(subStringOpType))
lexer.Add([]byte(`split`), opToken(splitStringOpType))
lexer.Add([]byte(`keys`), opToken(keysOpType))
@@ -324,15 +317,13 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`\]`), literalToken(closeCollect, true))
lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
lexer.Add([]byte(`\*[\+|\?d]*`), multiplyWithPrefs())
lexer.Add([]byte(`\*[\+|\?]*`), multiplyWithPrefs())
lexer.Add([]byte(`\+`), opToken(addOpType))
lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
lexer.Add([]byte(`\-`), opToken(subtractOpType))
lexer.Add([]byte(`\-=`), opToken(subtractAssignOpType))
lexer.Add([]byte(`\$[a-zA-Z_-0-9]+`), getVariableOpToken())
lexer.Add([]byte(`as`), opToken(assignVariableOpType))
err := lexer.CompileNFA()
err := lexer.Compile()
if err != nil {
return nil, err
}

View File

@@ -25,9 +25,6 @@ type operationType struct {
var orOpType = &operationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: orOperator}
var andOpType = &operationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: andOperator}
var reduceOpType = &operationType{Type: "REDUCE", NumArgs: 2, Precedence: 35, Handler: reduceOperator}
var blockOpType = &operationType{Type: "BLOCK", Precedence: 10, NumArgs: 2, Handler: emptyOperator}
var unionOpType = &operationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: unionOperator}
@@ -35,7 +32,6 @@ var pipeOpType = &operationType{Type: "PIPE", NumArgs: 2, Precedence: 30, Handle
var assignOpType = &operationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: assignUpdateOperator}
var addAssignOpType = &operationType{Type: "ADD_ASSIGN", NumArgs: 2, Precedence: 40, Handler: addAssignOperator}
var subtractAssignOpType = &operationType{Type: "SUBTRACT_ASSIGN", NumArgs: 2, Precedence: 40, Handler: subtractAssignOperator}
var assignAttributesOpType = &operationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: assignAttributesOperator}
var assignStyleOpType = &operationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: assignStyleOperator}
@@ -47,14 +43,11 @@ var assignAliasOpType = &operationType{Type: "ASSIGN_ALIAS", NumArgs: 2, Precede
var multiplyOpType = &operationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 42, Handler: multiplyOperator}
var addOpType = &operationType{Type: "ADD", NumArgs: 2, Precedence: 42, Handler: addOperator}
var subtractOpType = &operationType{Type: "SUBTRACT", NumArgs: 2, Precedence: 42, Handler: subtractOperator}
var alternativeOpType = &operationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 42, Handler: alternativeOperator}
var equalsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: equalsOperator}
var notEqualsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: notEqualsOperator}
//createmap needs to be above union, as we use union to build the components of the objects
var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 15, Handler: createMapOperator}
var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 40, Handler: createMapOperator}
var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: 45, Handler: pipeOperator}
@@ -75,7 +68,6 @@ var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50,
var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator}
var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}
var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator}
var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}
@@ -88,13 +80,14 @@ var selfReferenceOpType = &operationType{Type: "SELF", NumArgs: 0, Precedence: 5
var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: valueOperator}
var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator}
var notOpType = &operationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: notOperator}
var emptyOpType = &operationType{Type: "EMPTY", Precedence: 50, Handler: emptyOperator}
var emptyOpType = &operationType{Type: "EMPTY", NumArgs: 50, Handler: emptyOperator}
var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: recursiveDescentOperator}
var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator}
var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator}
var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
var deleteImmediateChildOpType = &operationType{Type: "DELETE_IMMEDIATE_CHILD", NumArgs: 1, Precedence: 40, Handler: deleteImmediateChildOperator}
type Operation struct {
OperationType *operationType

View File

@@ -16,9 +16,9 @@ func createAddOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {
func addAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
assignmentOp := &Operation{OperationType: assignOpType}
assignmentOp.UpdateAssign = true
selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createAddOp(selfExpression, expressionNode.Rhs)}
assignmentOp.UpdateAssign = false
assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createAddOp(expressionNode.Lhs, expressionNode.Rhs)}
return d.GetMatchingNodes(context, assignmentOpNode)
}
@@ -39,7 +39,7 @@ func toNodes(candidate *CandidateNode) []*yaml.Node {
func addOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("Add operator")
return crossFunction(d, context, expressionNode, add, false)
return crossFunction(d, context, expressionNode, add)
}
func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

View File

@@ -63,21 +63,13 @@ var addOperatorScenarios = []expressionScenario{
},
},
{
description: "Append to array",
description: "Update array (append)",
document: `{a: [1,2], b: [3,4]}`,
expression: `.a = .a + .b`,
expected: []string{
"D0, P[], (doc)::{a: [1, 2, 3, 4], b: [3, 4]}\n",
},
},
{
description: "Relative append",
document: `a: { a1: {b: [cat]}, a2: {b: [dog]}, a3: {} }`,
expression: `.a[].b += ["mouse"]`,
expected: []string{
"D0, P[], (doc)::a: {a1: {b: [cat, mouse]}, a2: {b: [dog, mouse]}, a3: {b: [mouse]}}\n",
},
},
{
description: "String concatenation",
document: `{a: cat, b: meow}`,
@@ -86,6 +78,14 @@ var addOperatorScenarios = []expressionScenario{
"D0, P[], (doc)::{a: catmeow, b: meow}\n",
},
},
{
description: "Relative string concatenation",
document: `{a: cat, b: meow}`,
expression: `.a += .b`,
expected: []string{
"D0, P[], (doc)::{a: catmeow, b: meow}\n",
},
},
{
description: "Number addition - float",
subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.",
@@ -105,11 +105,11 @@ var addOperatorScenarios = []expressionScenario{
},
},
{
description: "Increment numbers",
document: `{a: 3, b: 5}`,
expression: `.[] += 1`,
description: "Increment number",
document: `{a: 3}`,
expression: `.a += 1`,
expected: []string{
"D0, P[], (doc)::{a: 4, b: 6}\n",
"D0, P[], (doc)::{a: 4}\n",
},
},
{

View File

@@ -1,14 +1,14 @@
package yqlib
// corssFunction no matches
// can boolean use crossfunction
func alternativeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- alternative")
return crossFunction(d, context, expressionNode, alternativeFunc, true)
return crossFunction(d, context, expressionNode, alternativeFunc)
}
func alternativeFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
if lhs == nil {
return rhs, nil
}
lhs.Node = unwrapDoc(lhs.Node)
rhs.Node = unwrapDoc(rhs.Node)
log.Debugf("Alternative LHS: %v", lhs.Node.Tag)

View File

@@ -13,14 +13,6 @@ var alternativeOperatorScenarios = []expressionScenario{
"D0, P[a], (!!str)::bridge\n",
},
},
{
expression: `select(tag == "seq") // "cat"`,
skipDoc: true,
document: `a: frog`,
expected: []string{
"D0, P[], (!!str)::cat\n",
},
},
{
description: "LHS is not defined",
expression: `.a // "hello"`,

View File

@@ -143,5 +143,5 @@ func TestAssignOperatorScenarios(t *testing.T) {
for _, tt := range assignOperatorScenarios {
testScenario(t, &tt)
}
documentScenarios(t, "Assign (Update)", assignOperatorScenarios)
documentScenarios(t, "Assign", assignOperatorScenarios)
}

View File

@@ -49,7 +49,7 @@ func orOperator(d *dataTreeNavigator, context Context, expressionNode *Expressio
return crossFunction(d, context, expressionNode, performBoolOp(
func(b1 bool, b2 bool) bool {
return b1 || b2
}), false)
}))
}
func andOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@@ -57,7 +57,7 @@ func andOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
return crossFunction(d, context, expressionNode, performBoolOp(
func(b1 bool, b2 bool) bool {
return b1 && b2
}), false)
}))
}
func notOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {

View File

@@ -62,7 +62,7 @@ func sequenceFor(d *dataTreeNavigator, context Context, matchingNode *CandidateN
}
return &CandidateNode{Node: &node, Document: document, Path: path}, nil
}, false)
})
if err != nil {
return nil, err

View File

@@ -18,23 +18,48 @@ func deleteChildOperator(d *dataTreeNavigator, context Context, expressionNode *
for el := nodesToDelete.MatchingNodes.Back(); el != nil; el = el.Prev() {
candidate := el.Value.(*CandidateNode)
//problem: context may already be '.a' and then I pass in '.a.a2'.
// should pass in .a2.
if candidate.Parent == nil {
log.Info("Could not find parent of %v", candidate.GetKey())
return context, nil
if len(candidate.Path) > 0 {
deleteImmediateChildOp := &Operation{
OperationType: deleteImmediateChildOpType,
Value: candidate.Path[len(candidate.Path)-1],
}
deleteImmediateChildOpNode := &ExpressionNode{
Operation: deleteImmediateChildOp,
Rhs: createTraversalTree(candidate.Path[0:len(candidate.Path)-1], traversePreferences{}, false),
}
_, err := d.GetMatchingNodes(contextToUse, deleteImmediateChildOpNode)
if err != nil {
return Context{}, err
}
}
}
return context, nil
}
parentNode := candidate.Parent.Node
childPath := candidate.Path[len(candidate.Path)-1]
func deleteImmediateChildOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
parents, err := d.GetMatchingNodes(context, expressionNode.Rhs)
if err != nil {
return Context{}, err
}
childPath := expressionNode.Operation.Value
log.Debug("childPath to remove %v", childPath)
for el := parents.MatchingNodes.Front(); el != nil; el = el.Next() {
parent := el.Value.(*CandidateNode)
parentNode := unwrapDoc(parent.Node)
if parentNode.Kind == yaml.MappingNode {
deleteFromMap(candidate.Parent, childPath)
deleteFromMap(parent, childPath)
} else if parentNode.Kind == yaml.SequenceNode {
deleteFromArray(candidate.Parent, childPath)
deleteFromArray(parent, childPath)
} else {
return Context{}, fmt.Errorf("Cannot delete nodes from parent of tag %v", parentNode.Tag)
}
}
return context, nil
}

View File

@@ -21,70 +21,6 @@ var deleteOperatorScenarios = []expressionScenario{
"D0, P[], (doc)::{a: {a2: frood}}\n",
},
},
{
skipDoc: true,
document: `{a: {a1: fred, a2: frood}}`,
expression: `.a | del(.a1)`,
expected: []string{
"D0, P[a], (!!map)::{a2: frood}\n",
},
},
{
skipDoc: true,
document: `a: [1,2,3]`,
expression: `.a | del(.[1])`,
expected: []string{
"D0, P[a], (!!seq)::[1, 3]\n",
},
},
{
skipDoc: true,
document: `[0, {a: cat, b: dog}]`,
expression: `.[1] | del(.a)`,
expected: []string{
"D0, P[1], (!!map)::{b: dog}\n",
},
},
{
skipDoc: true,
document: `[{a: cat, b: dog}]`,
expression: `.[0] | del(.a)`,
expected: []string{
"D0, P[0], (!!map)::{b: dog}\n",
},
},
{
skipDoc: true,
document: `[{a: {b: thing, c: frog}}]`,
expression: `.[0].a | del(.b)`,
expected: []string{
"D0, P[0 a], (!!map)::{c: frog}\n",
},
},
{
skipDoc: true,
document: `[{a: {b: thing, c: frog}}]`,
expression: `.[0] | del(.a.b)`,
expected: []string{
"D0, P[0], (!!map)::{a: {c: frog}}\n",
},
},
{
skipDoc: true,
document: `{a: [0, {b: thing, c: frog}]}`,
expression: `.a[1] | del(.b)`,
expected: []string{
"D0, P[a 1], (!!map)::{c: frog}\n",
},
},
{
skipDoc: true,
document: `{a: [0, {b: thing, c: frog}]}`,
expression: `.a | del(.[1].b)`,
expected: []string{
"D0, P[a], (!!seq)::[0, {c: frog}]\n",
},
},
{
skipDoc: true,
document: `{a: {a1: fred, a2: frood}}`,

View File

@@ -4,7 +4,7 @@ import "gopkg.in/yaml.v3"
func equalsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- equalsOperation")
return crossFunction(d, context, expressionNode, isEquals(false), false)
return crossFunction(d, context, expressionNode, isEquals(false))
}
func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
@@ -29,5 +29,5 @@ func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *Candid
func notEqualsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- equalsOperation")
return crossFunction(d, context, expressionNode, isEquals(true), false)
return crossFunction(d, context, expressionNode, isEquals(true))
}

View File

@@ -21,16 +21,6 @@ var fileOperatorScenarios = []expressionScenario{
"D0, P[], (!!int)::0\n",
},
},
{
description: "Get file indices of multiple documents",
document: `{a: cat}`,
document2: `{a: cat}`,
expression: `fileIndex`,
expected: []string{
"D0, P[], (!!int)::0\n",
"D0, P[], (!!int)::1\n",
},
},
{
description: "Get file index alias",
document: `{a: cat}`,

View File

@@ -28,15 +28,6 @@ var hasOperatorScenarios = []expressionScenario{
"D0, P[3], (!!bool)::false\n",
},
},
{
description: "Select, checking for existence of deep paths",
subdescription: "Simply pipe in parent expressions into `has`",
document: "- {a: {b: {c: cat}}}\n- {a: {b: {d: dog}}}",
expression: `.[] | select(.a.b | has("c"))`,
expected: []string{
"D0, P[0], (!!map)::{a: {b: {c: cat}}}\n",
},
},
{
dontFormatInputForDoc: true,
description: "Has array index",

View File

@@ -10,14 +10,13 @@ import (
)
type multiplyPreferences struct {
AppendArrays bool
DeepMergeArrays bool
TraversePrefs traversePreferences
AppendArrays bool
TraversePrefs traversePreferences
}
func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- MultiplyOperator")
return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)), false)
return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)))
}
func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
@@ -31,7 +30,6 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
var newBlank = lhs.CreateChild(nil, &yaml.Node{})
var newThing, err = mergeObjects(d, context, newBlank, lhs, multiplyPreferences{})
if err != nil {
return nil, err
@@ -39,31 +37,11 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
return mergeObjects(d, context, newThing, rhs, preferences)
} else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" {
return multiplyIntegers(lhs, rhs)
} else if (lhs.Node.Tag == "!!int" || lhs.Node.Tag == "!!float") && (rhs.Node.Tag == "!!int" || rhs.Node.Tag == "!!float") {
return multiplyFloats(lhs, rhs)
}
return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
}
}
func multiplyFloats(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
target := lhs.CreateChild(nil, &yaml.Node{})
target.Node.Kind = yaml.ScalarNode
target.Node.Style = lhs.Node.Style
target.Node.Tag = "!!float"
lhsNum, err := strconv.ParseFloat(lhs.Node.Value, 64)
if err != nil {
return nil, err
}
rhsNum, err := strconv.ParseFloat(rhs.Node.Value, 64)
if err != nil {
return nil, err
}
target.Node.Value = fmt.Sprintf("%v", lhsNum*rhsNum)
return target, nil
}
func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
target := lhs.CreateChild(nil, &yaml.Node{})
target.Node.Kind = yaml.ScalarNode
@@ -120,12 +98,11 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
lhsPath := rhs.Path[pathIndexToStartFrom:]
assignmentOp := &Operation{OperationType: assignAttributesOpType}
if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
assignmentOp.OperationType = addAssignOpType
} else if !preferences.DeepMergeArrays && rhs.Node.Kind == yaml.SequenceNode ||
(rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode) {
if rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode {
assignmentOp.OperationType = assignOpType
assignmentOp.UpdateAssign = false
} else if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
assignmentOp.OperationType = addAssignOpType
}
rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}

View File

@@ -4,57 +4,7 @@ import (
"testing"
)
var doc1 = `list:
# Hi this is a comment.
# Hello this is another comment.
- "abc"`
var doc2 = `list2:
# This is yet another comment.
# Indeed this is yet another comment.
- "123"`
var docExpected = `D0, P[], (!!map)::list:
# Hi this is a comment.
# Hello this is another comment.
- "abc"
list2:
# This is yet another comment.
# Indeed this is yet another comment.
- "123"
`
var multiplyOperatorScenarios = []expressionScenario{
{
description: "Multiply integers",
expression: `3 * 4`,
expected: []string{
"D0, P[], (!!int)::12\n",
},
},
{
skipDoc: true,
document: doc1,
document2: doc2,
expression: `select(fi == 0) * select(fi == 1)`,
expected: []string{
docExpected,
},
},
{
skipDoc: true,
expression: `3 * 4.5`,
expected: []string{
"D0, P[], (!!float)::13.5\n",
},
},
{
skipDoc: true,
expression: `4.5 * 3`,
expected: []string{
"D0, P[], (!!float)::13.5\n",
},
},
{
skipDoc: true,
document: `{a: {also: [1]}, b: {also: me}}`,
@@ -207,7 +157,7 @@ var multiplyOperatorScenarios = []expressionScenario{
{
skipDoc: true,
document: `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,
expression: `.a *?d .b`,
expression: `.a *? .b`,
expected: []string{
"D0, P[a], (!!seq)::[{thing: two}]\n",
},
@@ -236,15 +186,6 @@ var multiplyOperatorScenarios = []expressionScenario{
"D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n",
},
},
{
description: "Merge, deeply merging arrays",
subdescription: "Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.",
document: `{a: [{name: fred, age: 12}, {name: bob, age: 32}], b: [{name: fred, age: 34}]}`,
expression: `.a *d .b`,
expected: []string{
"D0, P[a], (!!seq)::[{name: fred, age: 34}, {name: bob, age: 32}]\n",
},
},
{
description: "Merge to prefix an element",
document: `{a: cat, b: dog}`,
@@ -283,5 +224,5 @@ func TestMultiplyOperatorScenarios(t *testing.T) {
for _, tt := range multiplyOperatorScenarios {
testScenario(t, &tt)
}
documentScenarios(t, "Multiply (Merge)", multiplyOperatorScenarios)
documentScenarios(t, "Multiply", multiplyOperatorScenarios)
}

View File

@@ -1,59 +0,0 @@
package yqlib
import (
"container/list"
"fmt"
)
func reduceOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- reduceOp")
//.a as $var reduce (0; . + $var)
//lhs is the assignment operator
//rhs is the reduce block
// '.' refers to the current accumulator, initialised to 0
// $var references a single element from the .a
//ensure lhs is actually an assignment
//and rhs is a block (empty)
if expressionNode.Lhs.Operation.OperationType != assignVariableOpType {
return Context{}, fmt.Errorf("reduce must be given a variables assignment, got %v instead", expressionNode.Lhs.Operation.OperationType.Type)
} else if expressionNode.Rhs.Operation.OperationType != blockOpType {
return Context{}, fmt.Errorf("reduce must be given a block, got %v instead", expressionNode.Rhs.Operation.OperationType.Type)
}
arrayExpNode := expressionNode.Lhs.Lhs
array, err := d.GetMatchingNodes(context, arrayExpNode)
log.Debugf("array of %v things", array.MatchingNodes.Len())
if err != nil {
return Context{}, err
}
variableName := expressionNode.Lhs.Rhs.Operation.StringValue
initExp := expressionNode.Rhs.Lhs
accum, err := d.GetMatchingNodes(context, initExp)
if err != nil {
return Context{}, err
}
log.Debugf("with variable %v", variableName)
blockExp := expressionNode.Rhs.Rhs
for el := array.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
log.Debugf("REDUCING WITH %v", NodeToString(candidate))
l := list.New()
l.PushBack(candidate)
accum.SetVariable(variableName, l)
accum, err = d.GetMatchingNodes(accum, blockExp)
if err != nil {
return Context{}, err
}
}
return accum, nil
}

View File

@@ -1,40 +0,0 @@
package yqlib
import (
"testing"
)
var reduceOperatorScenarios = []expressionScenario{
{
description: "Sum numbers",
document: `[10,2, 5, 3]`,
expression: `.[] as $item ireduce (0; . + $item)`,
expected: []string{
"D0, P[], (!!int)::20\n",
},
},
{
description: "Merge all yaml files together",
document: `a: cat`,
document2: `b: dog`,
expression: `. as $item ireduce ({}; . * $item )`,
expected: []string{
"D0, P[], (!!map)::a: cat\nb: dog\n",
},
},
{
description: "Convert an array to an object",
document: `[{name: Cathy, has: apples},{name: Bob, has: bananas}]`,
expression: `.[] as $item ireduce ({}; .[$item | .name] = ($item | .has) )`,
expected: []string{
"D0, P[], (!!map)::Cathy: apples\nBob: bananas\n",
},
},
}
func TestReduceOperatorScenarios(t *testing.T) {
for _, tt := range reduceOperatorScenarios {
testScenario(t, &tt)
}
documentScenarios(t, "Reduce", reduceOperatorScenarios)
}

View File

@@ -3,77 +3,11 @@ package yqlib
import (
"container/list"
"fmt"
"regexp"
"strings"
"gopkg.in/yaml.v3"
)
func getSubstituteParameters(d *dataTreeNavigator, block *ExpressionNode, context Context) (string, string, error) {
regEx := ""
replacementText := ""
regExNodes, err := d.GetMatchingNodes(context, block.Lhs)
if err != nil {
return "", "", err
}
if regExNodes.MatchingNodes.Front() != nil {
regEx = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
}
log.Debug("regEx %v", regEx)
replacementNodes, err := d.GetMatchingNodes(context, block.Rhs)
if err != nil {
return "", "", err
}
if replacementNodes.MatchingNodes.Front() != nil {
replacementText = replacementNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
}
return regEx, replacementText, nil
}
func substitute(original string, regex *regexp.Regexp, replacement string) *yaml.Node {
replacedString := regex.ReplaceAllString(original, replacement)
return &yaml.Node{Kind: yaml.ScalarNode, Value: replacedString, Tag: "!!str"}
}
func substituteStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
//rhs block operator
//lhs of block = regex
//rhs of block = replacement expression
block := expressionNode.Rhs
regExStr, replacementText, err := getSubstituteParameters(d, block, context)
if err != nil {
return Context{}, err
}
regEx, err := regexp.Compile(regExStr)
if err != nil {
return Context{}, err
}
var results = list.New()
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
node := unwrapDoc(candidate.Node)
if node.Tag != "!!str" {
return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation.", node.Tag)
}
targetNode := substitute(node.Value, regEx, replacementText)
result := candidate.CreateChild(nil, targetNode)
results.PushBack(result)
}
return context.ChildContext(results), nil
}
func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- joinStringOperator")
joinStr := ""
@@ -92,7 +26,7 @@ func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *E
candidate := el.Value.(*CandidateNode)
node := unwrapDoc(candidate.Node)
if node.Kind != yaml.SequenceNode {
return Context{}, fmt.Errorf("cannot join with %v, can only join arrays of scalars", node.Tag)
return Context{}, fmt.Errorf("Cannot join with %v, can only join arrays of scalars", node.Tag)
}
targetNode := join(node.Content, joinStr)
result := candidate.CreateChild(nil, targetNode)

View File

@@ -13,24 +13,6 @@ var stringsOperatorScenarios = []expressionScenario{
"D0, P[], (!!str)::cat; meow; 1; ; true\n",
},
},
{
description: "Substitute / Replace string",
subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)\nNote the use of `|=` to run in context of the current string value.",
document: `a: dogs are great`,
expression: `.a |= sub("dogs", "cats")`,
expected: []string{
"D0, P[], (doc)::a: cats are great\n",
},
},
{
description: "Substitute / Replace string with regex",
subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)\nNote the use of `|=` to run in context of the current string value.",
document: "a: cat\nb: heat",
expression: `.[] |= sub("(a)", "${1}r")`,
expected: []string{
"D0, P[], (doc)::a: cart\nb: heart\n",
},
},
{
description: "Split strings",
document: `"cat; meow; 1; ; true"`,

View File

@@ -1,97 +0,0 @@
package yqlib
import (
"fmt"
"strconv"
yaml "gopkg.in/yaml.v3"
)
func createSubtractOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {
return &ExpressionNode{Operation: &Operation{OperationType: subtractOpType},
Lhs: lhs,
Rhs: rhs}
}
func subtractAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
assignmentOp := &Operation{OperationType: assignOpType}
assignmentOp.UpdateAssign = true
selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createSubtractOp(selfExpression, expressionNode.Rhs)}
return d.GetMatchingNodes(context, assignmentOpNode)
}
func subtractOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("Subtract operator")
return crossFunction(d, context, expressionNode, subtract, false)
}
func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
lhs.Node = unwrapDoc(lhs.Node)
rhs.Node = unwrapDoc(rhs.Node)
lhsNode := lhs.Node
if lhsNode.Tag == "!!null" {
return lhs.CreateChild(nil, rhs.Node), nil
}
target := lhs.CreateChild(nil, &yaml.Node{})
switch lhsNode.Kind {
case yaml.MappingNode:
return nil, fmt.Errorf("Maps not yet supported for subtraction")
case yaml.SequenceNode:
return nil, fmt.Errorf("Sequences not yet supported for subtraction")
// target.Node.Kind = yaml.SequenceNode
// target.Node.Style = lhsNode.Style
// target.Node.Tag = "!!seq"
// target.Node.Content = append(lhsNode.Content, toNodes(rhs)...)
case yaml.ScalarNode:
if rhs.Node.Kind != yaml.ScalarNode {
return nil, fmt.Errorf("%v (%v) cannot be added to a %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag)
}
target.Node.Kind = yaml.ScalarNode
target.Node.Style = lhsNode.Style
return subtractScalars(target, lhsNode, rhs.Node)
}
return target, nil
}
func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*CandidateNode, error) {
if lhs.Tag == "!!str" {
return nil, fmt.Errorf("strings cannot be subtracted")
} else if lhs.Tag == "!!int" && rhs.Tag == "!!int" {
lhsNum, err := strconv.Atoi(lhs.Value)
if err != nil {
return nil, err
}
rhsNum, err := strconv.Atoi(rhs.Value)
if err != nil {
return nil, err
}
result := lhsNum - rhsNum
target.Node.Tag = "!!int"
target.Node.Value = fmt.Sprintf("%v", result)
} else if (lhs.Tag == "!!int" || lhs.Tag == "!!float") && (rhs.Tag == "!!int" || rhs.Tag == "!!float") {
lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
if err != nil {
return nil, err
}
rhsNum, err := strconv.ParseFloat(rhs.Value, 64)
if err != nil {
return nil, err
}
result := lhsNum - rhsNum
target.Node.Tag = "!!float"
target.Node.Value = fmt.Sprintf("%v", result)
} else {
return nil, fmt.Errorf("%v cannot be added to %v", lhs.Tag, rhs.Tag)
}
return target, nil
}

View File

@@ -1,50 +0,0 @@
package yqlib
import (
"testing"
)
var subtractOperatorScenarios = []expressionScenario{
{
description: "Number subtraction - float",
subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.",
document: `{a: 3, b: 4.5}`,
expression: `.a = .a - .b`,
expected: []string{
"D0, P[], (doc)::{a: -1.5, b: 4.5}\n",
},
},
{
description: "Number subtraction - float",
subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.",
document: `{a: 3, b: 4.5}`,
expression: `.a = .a - .b`,
expected: []string{
"D0, P[], (doc)::{a: -1.5, b: 4.5}\n",
},
},
{
description: "Number subtraction - int",
subdescription: "If both the lhs and rhs are ints then the expression will be calculated with ints.",
document: `{a: 3, b: 4}`,
expression: `.a = .a - .b`,
expected: []string{
"D0, P[], (doc)::{a: -1, b: 4}\n",
},
},
{
description: "Decrement numbers",
document: `{a: 3, b: 5}`,
expression: `.[] -= 1`,
expected: []string{
"D0, P[], (doc)::{a: 2, b: 4}\n",
},
},
}
func TestSubtractOperatorScenarios(t *testing.T) {
for _, tt := range subtractOperatorScenarios {
testScenario(t, &tt)
}
documentScenarios(t, "Subtract", subtractOperatorScenarios)
}

View File

@@ -24,20 +24,7 @@ func emptyOperator(d *dataTreeNavigator, context Context, expressionNode *Expres
type crossFunctionCalculation func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)
func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *CandidateNode, rhs Context, calculation crossFunctionCalculation, results *list.List) error {
for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() {
log.Debugf("Applying calc")
rhsCandidate := rightEl.Value.(*CandidateNode)
resultCandidate, err := calculation(d, context, lhsCandidate, rhsCandidate)
if err != nil {
return err
}
results.PushBack(resultCandidate)
}
return nil
}
func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation, calcWhenEmpty bool) (Context, error) {
func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (Context, error) {
var results = list.New()
lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
if err != nil {
@@ -51,33 +38,24 @@ func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *Expressi
return Context{}, err
}
if calcWhenEmpty && lhs.MatchingNodes.Len() == 0 {
if rhs.MatchingNodes.Len() == 0 {
resultCandidate, err := calculation(d, context, nil, nil)
for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
lhsCandidate := el.Value.(*CandidateNode)
for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() {
log.Debugf("Applying calc")
rhsCandidate := rightEl.Value.(*CandidateNode)
resultCandidate, err := calculation(d, context, lhsCandidate, rhsCandidate)
if err != nil {
return Context{}, err
}
results.PushBack(resultCandidate)
}
err := resultsForRhs(d, context, nil, rhs, calculation, results)
if err != nil {
return Context{}, err
}
}
for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
lhsCandidate := el.Value.(*CandidateNode)
err := resultsForRhs(d, context, lhsCandidate, rhs, calculation, results)
if err != nil {
return Context{}, err
}
}
return context.ChildContext(results), nil
}
func crossFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation, calcWhenEmpty bool) (Context, error) {
func crossFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (Context, error) {
var results = list.New()
var evaluateAllTogether = true
@@ -88,11 +66,11 @@ func crossFunction(d *dataTreeNavigator, context Context, expressionNode *Expres
}
}
if evaluateAllTogether {
return doCrossFunc(d, context, expressionNode, calculation, calcWhenEmpty)
return doCrossFunc(d, context, expressionNode, calculation)
}
for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
innerResults, err := doCrossFunc(d, context.SingleChildContext(matchEl.Value.(*CandidateNode)), expressionNode, calculation, calcWhenEmpty)
innerResults, err := doCrossFunc(d, context.SingleChildContext(matchEl.Value.(*CandidateNode)), expressionNode, calculation)
if err != nil {
return Context{}, err
}

View File

@@ -34,7 +34,7 @@ func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEn
unwrapScalar: unwrapScalar,
colorsEnabled: colorsEnabled,
indent: indent,
printDocSeparators: !outputToJSON && printDocSeparators,
printDocSeparators: printDocSeparators,
firstTimePrinting: true,
treeNavigator: NewDataTreeNavigator(),
}

View File

@@ -130,9 +130,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
func TestPrinterMultipleDocsJson(t *testing.T) {
var output bytes.Buffer
var writer = bufio.NewWriter(&output)
// note printDocSeparators is true, it should still not print document separators
// when outputing JSON.
printer := NewPrinter(writer, true, true, false, 0, true)
printer := NewPrinter(writer, true, true, false, 0, false)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
if err != nil {

View File

@@ -2,7 +2,6 @@
- increment version in snapcraft.yaml
- increment version in github-action/Dockerfile
- make sure local build passes
- run ./scripts/secure.sh (manual because docker platforms)
- commit version update changes
- tag git with same version number
- commit vX tag - this will trigger github actions

View File

@@ -10,3 +10,12 @@ else
./bin/golangci-lint run --timeout=5m
fi
# ./bin/golangci-lint \
# --tests \
# --vendor \
# --disable=aligncheck \
# --disable=gotype \
# --disable=goconst \
# --disable=gocyclo \
# --deadline=300s \
# ./...

View File

@@ -1,5 +1,4 @@
#!/bin/sh
set -ex
go get golang.org/x/tools/cmd/goimports
wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.37.1
# wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.6.1
wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.24.0

View File

@@ -1,11 +0,0 @@
#!/bin/bash
set -o errexit
set -o pipefail
if command -v gosec &> /dev/null
then
gosec ${PWD}
else
./bin/gosec ${PWD}
fi

View File

@@ -1,5 +1,5 @@
name: yq
version: '4.7.1'
version: '4.5.1'
summary: A lightweight and portable command-line YAML processor
description: |
The aim of the project is to be the jq or sed of yaml files.

BIN
yq_linux_amd64.tar.gz Normal file

Binary file not shown.

BIN
yqt Executable file

Binary file not shown.