1
0
mirror of https://github.com/taigrr/yq synced 2025-01-18 04:53:17 -08:00

Compare commits

..

33 Commits

Author SHA1 Message Date
Mike Farah
8b327d0414 Increment version 2021-04-25 12:07:22 +10:00
Mike Farah
c8630fe4f3 Fixes delete issue #793 2021-04-25 12:05:56 +10:00
Mike Farah
87df9b1ae6 Updating operator doc 2021-04-24 17:41:06 +10:00
Mike Farah
2483c38eeb Added command help about using stdin 2021-04-19 10:27:01 +10:00
Mike Farah
b2a538bdfc Better string sub documentation 2021-04-16 16:07:40 +10:00
Mike Farah
6c26344449 Incrementing version 2021-04-15 16:22:24 +10:00
Mike Farah
daf0bfe1b9 Added string substitute command 2021-04-15 16:09:47 +10:00
Mike Farah
750a00ec35 Added "expand" to explode docs for searchability 2021-04-13 13:59:26 +10:00
Mike Farah
25e0a824c5 Fixed alternative operator when LHS has empty matches 2021-04-13 10:53:46 +10:00
Mike Farah
095f921f62 4.6.3 release 2021-03-25 08:45:59 +11:00
Mike Farah
12d3425b4a Added subtract operator (numbers only) 2021-03-25 08:12:01 +11:00
Mike Farah
0249f00bd5 Added chocolate badges thanks @adriens 2021-03-23 20:40:39 +11:00
Mike Farah
42b1bf9678 updated release notes to run gosec manually 2021-03-19 13:41:56 +11:00
Mike Farah
c632fd3641 cannot run gosec on all docker platforms, removing from devtools 2021-03-19 13:40:24 +11:00
Mike Farah
0470e5cd69 cannot run gosec on all docker platforms, removing for now 2021-03-19 13:21:07 +11:00
Mike Farah
35c6d07248 Bump version 2021-03-19 12:59:00 +11:00
Mike Farah
3f0be107d4 Bump dependencies 2021-03-19 12:58:43 +11:00
Mike Farah
21a9e506cb Fixed merge comments 2021-03-19 12:54:03 +11:00
Mike Farah
3722367fbb Dont print doc separators for JSON (https://github.com/mikefarah/yq/issues/735) 2021-03-19 12:40:56 +11:00
Mike Farah
f7b50e9853 Fixed += operator (https://github.com/mikefarah/yq/issues/750) 2021-03-19 12:36:05 +11:00
Mike Farah
4f3fe256aa Fixed precedence of CREATE_MAP (https://github.com/mikefarah/yq/issues/753) 2021-03-19 12:09:32 +11:00
Mike Farah
2595a929c9 Fixing doc links 2021-03-19 11:52:13 +11:00
Mike Farah
b2186d5404 Added gosec 2021-03-03 19:44:34 +11:00
Mike Farah
e93c43f7a0 Improving docs 2021-02-26 11:31:43 +11:00
Mike Farah
b1a5387cdd Update readme 2021-02-25 17:04:08 +11:00
Mike Farah
5911ab2929 Bump version 2021-02-25 16:52:49 +11:00
Mike Farah
2ed5b2ff59 Improved lexer performance! 2021-02-25 16:47:55 +11:00
Mike Farah
111c6e0be1 Increment version 2021-02-18 11:19:16 +11:00
Mike Farah
81136ad57e Arrays no longer deeply merge by defauly, like jq 2021-02-18 11:16:54 +11:00
Mike Farah
a6cd250987 nicer reduce example 2021-02-15 18:23:50 +11:00
Mike Farah
ee1f55630f nicer reduce example 2021-02-15 17:33:41 +11:00
Mike Farah
9072e8d3b3 Added context variable for reduce 2021-02-15 17:31:12 +11:00
Mike Farah
99b08fd612 Added reduce examples and doc 2021-02-15 16:38:53 +11:00
59 changed files with 1015 additions and 217 deletions

View File

@@ -1 +1 @@
bin
bin/*

View File

@@ -6,26 +6,5 @@ RUN set -e -x \
&& /opt/devtools.sh
ENV PATH=/go/bin:$PATH
# install mkdocs
RUN set -ex \
&& buildDeps=' \
build-essential \
python3-dev \
' \
&& apt-get update && apt-get install -y --no-install-recommends \
$buildDeps \
python3 \
python3-setuptools \
python3-wheel \
python3-pip \
&& pip3 install --upgrade \
pip \
'Markdown>=2.6.9' \
'mkdocs>=0.16.3' \
'mkdocs-material>=1.10.1' \
'markdown-include>=0.5.1' \
&& apt-get purge -y --auto-remove $buildDeps \
&& rm -rf /var/lib/apt/lists/*
ENV CGO_ENABLED 0
ENV GOPATH /go:/yq

View File

@@ -17,6 +17,7 @@ help:
@echo ' make vendor Install dependencies to vendor directory.'
@echo ' make format Run code formatter.'
@echo ' make check Run static code analysis (lint).'
@echo ' make secure Run gosec.'
@echo ' make test Run tests on project.'
@echo ' make cover Run tests and capture code coverage metrics on project.'
@echo ' make clean Clean the directory tree of produced artifacts.'
@@ -84,6 +85,10 @@ format: vendor
check: format
${DOCKRUN} bash ./scripts/check.sh
.PHONY: secure
secure:
${DOCKRUN} bash ./scripts/secure.sh
.PHONY: test
test: check
${DOCKRUN} bash ./scripts/test.sh
@@ -96,11 +101,6 @@ cover: check
@find cover -type d -exec chmod 755 {} \; || :
@find cover -type f -exec chmod 644 {} \; || :
.PHONY: build-docs
build-docs: prepare mkdocs.yml mkdocs/*
${DOCKRUN} mkdocs build
@find docs -type d -exec chmod 755 {} \; || :
@find docs -type f -exec chmod 644 {} \; || :
.PHONY: release
release: xcompile

View File

@@ -119,6 +119,8 @@ See [webi](https://webinstall.dev/)
Supported by @adithyasunil26 (https://github.com/webinstall/webi-installers/tree/master/yq)
### Windows:
[![Chocolatey](https://img.shields.io/chocolatey/v/yq.svg)](https://chocolatey.org/packages/yq)
[![Chocolatey](https://img.shields.io/chocolatey/dt/yq.svg)](https://chocolatey.org/packages/yq)
```
choco install yq
```
@@ -151,19 +153,21 @@ sudo apt install yq -y
Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
## Features
- [Detailed documentation with many examples](https://mikefarah.gitbook.io/yq/)
- Written in portable go, so you can download a lovely dependency free binary
- Uses similar syntax as `jq` but works with YAML and JSON files
- Fully supports multi document yaml files
- Colorized yaml output
- [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/v/v4.x/traverse)
- [Sort yaml by keys](https://mikefarah.gitbook.io/yq/v/v4.x/sort-keys)
- Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/comment-operators), [styling](https://mikefarah.gitbook.io/yq/style), [tags](https://mikefarah.gitbook.io/yq/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/anchor-and-alias-operators).
- [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/operators/traverse-read)
- [Sort yaml by keys](https://mikefarah.gitbook.io/yq/operators/sort-keys)
- Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/operators/comment-operators), [styling](https://mikefarah.gitbook.io/yq/operators/style), [tags](https://mikefarah.gitbook.io/yq/operators/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators).
- [Update yaml inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
- [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/v/v4.x/select#select-and-update-matching-values-in-map)
- [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map)
- Keeps yaml formatting and comments when updating (though there are issues with whitespace)
- [Convert to/from json to yaml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
- [Pipe data in by using '-'](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate)
- [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
- [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things.
## [Usage](https://mikefarah.gitbook.io/yq/)

View File

@@ -17,6 +17,9 @@ func createEvaluateAllCommand() *cobra.Command {
Example: `
# merges f2.yml into f1.yml (inplace)
yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml
# use '-' as a filename to read from STDIN
cat file2.yml | yq ea '.a.b' file1.yml - file3.yml
`,
Long: "Evaluate All:\nUseful when you need to run an expression across several yaml documents or files. Consumes more memory than eval",
RunE: evaluateAll,

View File

@@ -21,6 +21,9 @@ yq e '.a.b | length' f1.yml f2.yml
# prints out the file
yq e sample.yaml
# use '-' as a filename to read from STDIN
cat file2.yml | yq e '.a.b' file1.yml - file3.yml
# prints a new yaml document
yq e -n '.a.b.c = "cat"'

View File

@@ -11,7 +11,7 @@ var (
GitDescribe string
// Version is main version number that is being run at the moment.
Version = "4.5.1"
Version = "4.7.1"
// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release

View File

@@ -1,4 +1,4 @@
FROM mikefarah/yq:4.5.1
FROM mikefarah/yq:4.7.1
COPY entrypoint.sh /entrypoint.sh

8
go.mod
View File

@@ -3,12 +3,12 @@ module github.com/mikefarah/yq/v4
require (
github.com/elliotchance/orderedmap v1.4.0
github.com/fatih/color v1.10.0
github.com/goccy/go-yaml v1.8.8
github.com/jinzhu/copier v0.2.3
github.com/spf13/cobra v1.1.1
github.com/goccy/go-yaml v1.8.9
github.com/jinzhu/copier v0.2.8
github.com/spf13/cobra v1.1.3
github.com/timtadh/data-structures v0.5.3 // indirect
github.com/timtadh/lexmachine v0.2.2
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e // indirect
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)

26
go.sum
View File

@@ -52,8 +52,8 @@ github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTM
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/goccy/go-yaml v1.8.8 h1:MGfRB1GeSn/hWXYWS2Pt67iC2GJNnebdIro01ddyucA=
github.com/goccy/go-yaml v1.8.8/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
github.com/goccy/go-yaml v1.8.9 h1:4AEXg2qx+/w29jXnXpMY6mTckmYu1TMoHteKuMf0HFg=
github.com/goccy/go-yaml v1.8.9/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -101,8 +101,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jinzhu/copier v0.2.3 h1:Oe09ju+9qft7TffZ7l/04AB2f8u1+V4ZMxmp/nnqeOs=
github.com/jinzhu/copier v0.2.3/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
github.com/jinzhu/copier v0.2.8 h1:N8MbL5niMwE3P4dOwurJixz5rMkKfujmMRFmAanSzWE=
github.com/jinzhu/copier v0.2.8/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -168,10 +168,9 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@@ -180,7 +179,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -257,10 +255,9 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e h1:XNp2Flc/1eWQGk5BLzqTAN7fQIwIbfyVTuVxXxZh73M=
golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -270,7 +267,6 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -283,7 +279,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
@@ -310,7 +305,6 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -321,11 +315,9 @@ gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -8,9 +8,10 @@ import (
)
type CandidateNode struct {
Node *yaml.Node // the actual node
Path []interface{} /// the path we took to get to this node
Document uint // the document index of this node
Node *yaml.Node // the actual node
Parent *CandidateNode // parent node
Path []interface{} /// the path we took to get to this node
Document uint // the document index of this node
Filename string
FileIndex int
// when performing op against all nodes given, this will treat all the nodes as one
@@ -31,6 +32,7 @@ func (n *CandidateNode) CreateChild(path interface{}, node *yaml.Node) *Candidat
return &CandidateNode{
Node: node,
Path: n.createChildPath(path),
Parent: n,
Document: n.Document,
Filename: n.Filename,
FileIndex: n.FileIndex,
@@ -86,7 +88,14 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) {
if n.Node.Style == 0 {
n.Node.Style = other.Node.Style
}
n.Node.FootComment = n.Node.FootComment + other.Node.FootComment
n.Node.HeadComment = n.Node.HeadComment + other.Node.HeadComment
n.Node.LineComment = n.Node.LineComment + other.Node.LineComment
if other.Node.FootComment != "" {
n.Node.FootComment = other.Node.FootComment
}
if other.Node.HeadComment != "" {
n.Node.HeadComment = other.Node.HeadComment
}
if other.Node.LineComment != "" {
n.Node.LineComment = other.Node.LineComment
}
}

View File

@@ -101,7 +101,7 @@ will output
- hello
```
## Update array (append)
## Append to array
Given a sample.yml file of:
```yaml
a:
@@ -127,6 +127,36 @@ b:
- 4
```
## Relative append
Given a sample.yml file of:
```yaml
a:
a1:
b:
- cat
a2:
b:
- dog
a3: {}
```
then
```bash
yq eval '.a[].b += ["mouse"]' sample.yml
```
will output
```yaml
a:
a1:
b:
- cat
- mouse
a2:
b:
- dog
- mouse
a3: {b: [mouse]}
```
## String concatenation
Given a sample.yml file of:
```yaml
@@ -143,22 +173,6 @@ a: catmeow
b: meow
```
## Relative string concatenation
Given a sample.yml file of:
```yaml
a: cat
b: meow
```
then
```bash
yq eval '.a += .b' sample.yml
```
will output
```yaml
a: catmeow
b: meow
```
## Number addition - float
If the lhs or rhs are floats then the expression will be calculated with floats.
@@ -195,18 +209,20 @@ a: 7
b: 4
```
## Increment number
## Increment numbers
Given a sample.yml file of:
```yaml
a: 3
b: 5
```
then
```bash
yq eval '.a += 1' sample.yml
yq eval '.[] += 1' sample.yml
```
will output
```yaml
a: 4
b: 6
```
## Add to null

View File

@@ -1,4 +1,4 @@
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference aliases and remove anchor names).
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.

View File

@@ -1,10 +1,4 @@
This operator is used to update node values. It can be used in either the:
### plain form: `=`
Which will assign the LHS node values to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.
### relative form: `|=`
This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment.
## Create yaml file
Running
```bash

View File

@@ -1,4 +1,15 @@
Use these comment operators to set or retrieve comments.
Like the `=` and `|=` assign operators, the same syntax applies when updating comments:
### plain form: `=`
This will assign the LHS nodes comments to the expression on the RHS. The RHS is run against the matching nodes in the pipeline
### relative form: `|=`
Similar to the plain form, however the RHS evaluates against each matching LHS node! This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.
## Set line comment
Given a sample.yml file of:
```yaml

View File

@@ -35,6 +35,26 @@ will output
0
```
## Get file indices of multiple documents
Given a sample.yml file of:
```yaml
a: cat
```
And another sample another.yml file of:
```yaml
a: cat
```
then
```bash
yq eval-all 'fileIndex' sample.yml another.yml
```
will output
```yaml
0
---
1
```
## Get file index alias
Given a sample.yml file of:
```yaml

View File

@@ -19,6 +19,29 @@ true
false
```
## Select, checking for existence of deep paths
Simply pipe in parent expressions into `has`
Given a sample.yml file of:
```yaml
- a:
b:
c: cat
- a:
b:
d: dog
```
then
```bash
yq eval '.[] | select(.a.b | has("c"))' sample.yml
```
will output
```yaml
a:
b:
c: cat
```
## Has array index
Given a sample.yml file of:
```yaml

View File

@@ -1,19 +1,34 @@
Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.
To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
## Objects and arrays - merging
Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged.
To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?.
Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
Multiplication of strings and numbers are not yet supported.
### Merge Flags
You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`. See examples below
## Merging files
- `+` to append arrays
- `?` to only merge existing fields
- `d` to deeply merge arrays
### Merging files
Note the use of `eval-all` to ensure all documents are loaded into memory.
```bash
yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml
```
## Multiply integers
Running
```bash
yq eval --null-input '3 * 4'
```
will output
```yaml
12
```
## Merge objects together, returning merged result only
Given a sample.yml file of:
```yaml
@@ -190,6 +205,32 @@ thing:
- 4
```
## Merge, deeply merging arrays
Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.
Given a sample.yml file of:
```yaml
a:
- name: fred
age: 12
- name: bob
age: 32
b:
- name: fred
age: 34
```
then
```bash
yq eval '.a *d .b' sample.yml
```
will output
```yaml
- name: fred
age: 34
- name: bob
age: 32
```
## Merge to prefix an element
Given a sample.yml file of:
```yaml

View File

@@ -0,0 +1,76 @@
Reduce is a powerful way to process a collection of data into a new form.
```
<exp> as $<name> ireduce (<init>; <block>)
```
e.g.
```
.[] as $item ireduce (0; . + $item)
```
On the LHS we are configuring the collection of items that will be reduced `<exp>` as well as what each element will be called `$<name>`. Note that the array has been splatted into its individual elements.
On the RHS there is `<init>`, the starting value of the accumulator and `<block>`, the expression that will update the accumulator for each element in the collection. Note that within the block expression, `.` will evaluate to the current value of the accumulator.
## yq vs jq syntax
Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`).
To that end, the reduce operator is called `ireduce` for backwards compatability if a `jq` like prefix version of `reduce` is ever added.
## Sum numbers
Given a sample.yml file of:
```yaml
- 10
- 2
- 5
- 3
```
then
```bash
yq eval '.[] as $item ireduce (0; . + $item)' sample.yml
```
will output
```yaml
20
```
## Merge all yaml files together
Given a sample.yml file of:
```yaml
a: cat
```
And another sample another.yml file of:
```yaml
b: dog
```
then
```bash
yq eval-all '. as $item ireduce ({}; . * $item )' sample.yml another.yml
```
will output
```yaml
a: cat
b: dog
```
## Convert an array to an object
Given a sample.yml file of:
```yaml
- name: Cathy
has: apples
- name: Bob
has: bananas
```
then
```bash
yq eval '.[] as $item ireduce ({}; .[$item | .name] = ($item | .has) )' sample.yml
```
will output
```yaml
Cathy: apples
Bob: bananas
```

View File

@@ -18,6 +18,42 @@ will output
cat; meow; 1; ; true
```
## Substitute / Replace string
This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
Note the use of `|=` to run in context of the current string value.
Given a sample.yml file of:
```yaml
a: dogs are great
```
then
```bash
yq eval '.a |= sub("dogs", "cats")' sample.yml
```
will output
```yaml
a: cats are great
```
## Substitute / Replace string with regex
This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
Note the use of `|=` to run in context of the current string value.
Given a sample.yml file of:
```yaml
a: cat
b: heat
```
then
```bash
yq eval '.[] |= sub("(a)", "${1}r")' sample.yml
```
will output
```yaml
a: cart
b: heart
```
## Split strings
Given a sample.yml file of:
```yaml

71
pkg/yqlib/doc/Subtract.md Normal file
View File

@@ -0,0 +1,71 @@
## Number subtraction - float
If the lhs or rhs are floats then the expression will be calculated with floats.
Given a sample.yml file of:
```yaml
a: 3
b: 4.5
```
then
```bash
yq eval '.a = .a - .b' sample.yml
```
will output
```yaml
a: -1.5
b: 4.5
```
## Number subtraction - float
If the lhs or rhs are floats then the expression will be calculated with floats.
Given a sample.yml file of:
```yaml
a: 3
b: 4.5
```
then
```bash
yq eval '.a = .a - .b' sample.yml
```
will output
```yaml
a: -1.5
b: 4.5
```
## Number subtraction - int
If both the lhs and rhs are ints then the expression will be calculated with ints.
Given a sample.yml file of:
```yaml
a: 3
b: 4
```
then
```bash
yq eval '.a = .a - .b' sample.yml
```
will output
```yaml
a: -1
b: 4
```
## Decrement numbers
Given a sample.yml file of:
```yaml
a: 3
b: 5
```
then
```bash
yq eval '.[] -= 1' sample.yml
```
will output
```yaml
a: 2
b: 4
```

View File

@@ -1,10 +1,44 @@
# Operators
In `yq` expressions are made up of operators. Operators have 0-2 arguments and run against the current 'matching' nodes in the expression tree.
In `yq` expressions are made up of operators and pipes. A context of nodes is passed through the expression and each operation takes the context as input and returns a new context as output. That output is piped in as input for the next operation in the expression. To begin with, the context is set to the first yaml document of the first yaml file (if processing in sequence using eval).
Lets look at a couple of examples.
The `length` operator take no arguments, and will simply return the length of _each_ matching node. So if there were 2 nodes, one string and one array, length will update the 'matching' nodes context to be two new numeric scalar nodes representing the lengths of the orignal 'matching' nodes.
## Example 1 - simple example
Given a document like:
```yaml
- [a]
- "cat"
```
with an expression:
```
.[] | length
```
`yq` will initially set the context as single node of the entire yaml document, an array of two elements.
```yaml
- [a]
- "cat"
```
This gets piped into the splat operator `.[]` which will split out the context into a collection of two nodes `[a]` and `"cat"`. Note that this is _not_ a yaml array.
The `length` operator take no arguments, and will simply return the length of _each_ matching node in the context. So for the context of `[a]` and `"cat"`, it will return a new context of `1` and `3`.
This being the last operation in the expression, the results will be printed out:
```
1
3
```
# Example 2 - operators with arguments.
The `=` operator takes two arguments, a `lhs` expression and `rhs` expression. It runs the 'matching' nodes context against the `lhs` expression to find the nodes to update, lets call it `lhsNodes`, and then runs the matching nodes against the `rhs` to find the new values, lets call that `rhsNodes`. It updates the `lhsNodes` values with the `rhsNodes` values and _returns the original matching nodes_. This is important, where length changed the matching nodes to be new nodes with the length values, `=` returns the original matching nodes, albeit with some of the nodes values updated. So `.a = 3` will still return the parent matching node, but with the matching child updated.

View File

@@ -1,4 +1,4 @@
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference aliases and remove anchor names).
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.

View File

@@ -1 +1,11 @@
Use these comment operators to set or retrieve comments.
Use these comment operators to set or retrieve comments.
Like the `=` and `|=` assign operators, the same syntax applies when updating comments:
### plain form: `=`
This will assign the LHS nodes comments to the expression on the RHS. The RHS is run against the matching nodes in the pipeline
### relative form: `|=`
Similar to the plain form, however the RHS evaluates against each matching LHS node! This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.

View File

@@ -0,0 +1,20 @@
Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.
## Objects and arrays - merging
Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged.
Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
### Merge Flags
You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`. See examples below
- `+` to append arrays
- `?` to only merge existing fields
- `d` to deeply merge arrays
### Merging files
Note the use of `eval-all` to ensure all documents are loaded into memory.
```bash
yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml
```

View File

@@ -1,15 +0,0 @@
Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?.
Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
Multiplication of strings and numbers are not yet supported.
## Merging files
Note the use of `eval-all` to ensure all documents are loaded into memory.
```bash
yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml
```

View File

@@ -0,0 +1,21 @@
Reduce is a powerful way to process a collection of data into a new form.
```
<exp> as $<name> ireduce (<init>; <block>)
```
e.g.
```
.[] as $item ireduce (0; . + $item)
```
On the LHS we are configuring the collection of items that will be reduced `<exp>` as well as what each element will be called `$<name>`. Note that the array has been splatted into its individual elements.
On the RHS there is `<init>`, the starting value of the accumulator and `<block>`, the expression that will update the accumulator for each element in the collection. Note that within the block expression, `.` will evaluate to the current value of the accumulator.
## yq vs jq syntax
Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`).
To that end, the reduce operator is called `ireduce` for backwards compatability if a `jq` like prefix version of `reduce` is ever added.

View File

@@ -5,9 +5,6 @@ import (
"strings"
)
var myPathTokeniser = newExpressionTokeniser()
var myPathPostfixer = newExpressionPostFixer()
type ExpressionNode struct {
Operation *Operation
Lhs *ExpressionNode
@@ -19,19 +16,21 @@ type ExpressionParser interface {
}
type expressionParserImpl struct {
pathTokeniser expressionTokeniser
pathPostFixer expressionPostFixer
}
func NewExpressionParser() ExpressionParser {
return &expressionParserImpl{}
return &expressionParserImpl{newExpressionTokeniser(), newExpressionPostFixer()}
}
func (p *expressionParserImpl) ParseExpression(expression string) (*ExpressionNode, error) {
tokens, err := myPathTokeniser.Tokenise(expression)
tokens, err := p.pathTokeniser.Tokenise(expression)
if err != nil {
return nil, err
}
var Operations []*Operation
Operations, err = myPathPostfixer.ConvertToPostfix(tokens)
Operations, err = p.pathPostFixer.ConvertToPostfix(tokens)
if err != nil {
return nil, err
}

View File

@@ -12,6 +12,21 @@ var pathTests = []struct {
expectedTokens []interface{}
expectedPostFix []interface{}
}{
{
`[]|join(".")`,
append(make([]interface{}, 0), "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")"),
append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE", ". (string)", "JOIN", "PIPE"),
},
{
`{"cool": .b or .c}`,
append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "b", "OR", "c", "}"),
append(make([]interface{}, 0), "cool (string)", "b", "c", "OR", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
},
{
`{"cool": []|join(".")}`,
append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")", "}"),
append(make([]interface{}, 0), "cool (string)", "EMPTY", "COLLECT", "SHORT_PIPE", ". (string)", "JOIN", "PIPE", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
},
{
`.a as $item ireduce (0; . + $item)`, // note - add code to shuffle reduce to this position for postfix
append(make([]interface{}, 0), "a", "ASSIGN_VARIABLE", "GET_VARIABLE", "REDUCE", "(", "0 (int64)", "BLOCK", "SELF", "ADD", "GET_VARIABLE", ")"),

View File

@@ -100,6 +100,9 @@ func multiplyWithPrefs() lex.Action {
if strings.Contains(options, "?") {
prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
}
if strings.Contains(options, "d") {
prefs.DeepMergeArrays = true
}
op := &Operation{OperationType: multiplyOpType, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs}
return &token{TokenType: operationToken, Operation: op}, nil
}
@@ -186,7 +189,7 @@ func getVariableOpToken() lex.Action {
return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
value := string(m.Bytes)
value = value[1 : len(value)-1]
value = value[1:]
getVarOperation := createValueOperation(value, value)
getVarOperation.OperationType = getVariableOpType
@@ -261,6 +264,8 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`splitDoc`), opToken(splitDocumentOpType))
lexer.Add([]byte(`join`), opToken(joinStringOpType))
lexer.Add([]byte(`sub`), opToken(subStringOpType))
lexer.Add([]byte(`split`), opToken(splitStringOpType))
lexer.Add([]byte(`keys`), opToken(keysOpType))
@@ -319,13 +324,15 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`\]`), literalToken(closeCollect, true))
lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
lexer.Add([]byte(`\*[\+|\?]*`), multiplyWithPrefs())
lexer.Add([]byte(`\*[\+|\?d]*`), multiplyWithPrefs())
lexer.Add([]byte(`\+`), opToken(addOpType))
lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
lexer.Add([]byte(`\-`), opToken(subtractOpType))
lexer.Add([]byte(`\-=`), opToken(subtractAssignOpType))
lexer.Add([]byte(`\$[a-zA-Z_-0-9]+`), getVariableOpToken())
lexer.Add([]byte(`as`), opToken(assignVariableOpType))
err := lexer.Compile()
err := lexer.CompileNFA()
if err != nil {
return nil, err
}

View File

@@ -25,7 +25,7 @@ type operationType struct {
var orOpType = &operationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: orOperator}
var andOpType = &operationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: andOperator}
var reduceOpType = &operationType{Type: "REDUCE", NumArgs: 2, Precedence: 5, Handler: reduceOperator}
var reduceOpType = &operationType{Type: "REDUCE", NumArgs: 2, Precedence: 35, Handler: reduceOperator}
var blockOpType = &operationType{Type: "BLOCK", Precedence: 10, NumArgs: 2, Handler: emptyOperator}
@@ -35,6 +35,7 @@ var pipeOpType = &operationType{Type: "PIPE", NumArgs: 2, Precedence: 30, Handle
var assignOpType = &operationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: assignUpdateOperator}
var addAssignOpType = &operationType{Type: "ADD_ASSIGN", NumArgs: 2, Precedence: 40, Handler: addAssignOperator}
var subtractAssignOpType = &operationType{Type: "SUBTRACT_ASSIGN", NumArgs: 2, Precedence: 40, Handler: subtractAssignOperator}
var assignAttributesOpType = &operationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: assignAttributesOperator}
var assignStyleOpType = &operationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: assignStyleOperator}
@@ -46,11 +47,14 @@ var assignAliasOpType = &operationType{Type: "ASSIGN_ALIAS", NumArgs: 2, Precede
var multiplyOpType = &operationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 42, Handler: multiplyOperator}
var addOpType = &operationType{Type: "ADD", NumArgs: 2, Precedence: 42, Handler: addOperator}
var subtractOpType = &operationType{Type: "SUBTRACT", NumArgs: 2, Precedence: 42, Handler: subtractOperator}
var alternativeOpType = &operationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 42, Handler: alternativeOperator}
var equalsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: equalsOperator}
var notEqualsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: notEqualsOperator}
var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 40, Handler: createMapOperator}
//createmap needs to be above union, as we use union to build the components of the objects
var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 15, Handler: createMapOperator}
var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: 45, Handler: pipeOperator}
@@ -71,6 +75,7 @@ var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50,
var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator}
var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}
var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator}
var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}
@@ -90,7 +95,6 @@ var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs:
var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator}
var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator}
var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
var deleteImmediateChildOpType = &operationType{Type: "DELETE_IMMEDIATE_CHILD", NumArgs: 1, Precedence: 40, Handler: deleteImmediateChildOperator}
type Operation struct {
OperationType *operationType

View File

@@ -16,9 +16,9 @@ func createAddOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {
func addAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
assignmentOp := &Operation{OperationType: assignOpType}
assignmentOp.UpdateAssign = false
assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createAddOp(expressionNode.Lhs, expressionNode.Rhs)}
assignmentOp.UpdateAssign = true
selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createAddOp(selfExpression, expressionNode.Rhs)}
return d.GetMatchingNodes(context, assignmentOpNode)
}
@@ -39,7 +39,7 @@ func toNodes(candidate *CandidateNode) []*yaml.Node {
func addOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("Add operator")
return crossFunction(d, context, expressionNode, add)
return crossFunction(d, context, expressionNode, add, false)
}
func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

View File

@@ -63,7 +63,7 @@ var addOperatorScenarios = []expressionScenario{
},
},
{
description: "Update array (append)",
description: "Append to array",
document: `{a: [1,2], b: [3,4]}`,
expression: `.a = .a + .b`,
expected: []string{
@@ -71,17 +71,17 @@ var addOperatorScenarios = []expressionScenario{
},
},
{
description: "String concatenation",
document: `{a: cat, b: meow}`,
expression: `.a = .a + .b`,
description: "Relative append",
document: `a: { a1: {b: [cat]}, a2: {b: [dog]}, a3: {} }`,
expression: `.a[].b += ["mouse"]`,
expected: []string{
"D0, P[], (doc)::{a: catmeow, b: meow}\n",
"D0, P[], (doc)::a: {a1: {b: [cat, mouse]}, a2: {b: [dog, mouse]}, a3: {b: [mouse]}}\n",
},
},
{
description: "Relative string concatenation",
description: "String concatenation",
document: `{a: cat, b: meow}`,
expression: `.a += .b`,
expression: `.a = .a + .b`,
expected: []string{
"D0, P[], (doc)::{a: catmeow, b: meow}\n",
},
@@ -105,11 +105,11 @@ var addOperatorScenarios = []expressionScenario{
},
},
{
description: "Increment number",
document: `{a: 3}`,
expression: `.a += 1`,
description: "Increment numbers",
document: `{a: 3, b: 5}`,
expression: `.[] += 1`,
expected: []string{
"D0, P[], (doc)::{a: 4}\n",
"D0, P[], (doc)::{a: 4, b: 6}\n",
},
},
{

View File

@@ -1,14 +1,14 @@
package yqlib
// corssFunction no matches
// can boolean use crossfunction
func alternativeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- alternative")
return crossFunction(d, context, expressionNode, alternativeFunc)
return crossFunction(d, context, expressionNode, alternativeFunc, true)
}
func alternativeFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
if lhs == nil {
return rhs, nil
}
lhs.Node = unwrapDoc(lhs.Node)
rhs.Node = unwrapDoc(rhs.Node)
log.Debugf("Alternative LHS: %v", lhs.Node.Tag)

View File

@@ -13,6 +13,14 @@ var alternativeOperatorScenarios = []expressionScenario{
"D0, P[a], (!!str)::bridge\n",
},
},
{
expression: `select(tag == "seq") // "cat"`,
skipDoc: true,
document: `a: frog`,
expected: []string{
"D0, P[], (!!str)::cat\n",
},
},
{
description: "LHS is not defined",
expression: `.a // "hello"`,

View File

@@ -143,5 +143,5 @@ func TestAssignOperatorScenarios(t *testing.T) {
for _, tt := range assignOperatorScenarios {
testScenario(t, &tt)
}
documentScenarios(t, "Assign", assignOperatorScenarios)
documentScenarios(t, "Assign (Update)", assignOperatorScenarios)
}

View File

@@ -49,7 +49,7 @@ func orOperator(d *dataTreeNavigator, context Context, expressionNode *Expressio
return crossFunction(d, context, expressionNode, performBoolOp(
func(b1 bool, b2 bool) bool {
return b1 || b2
}))
}), false)
}
func andOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@@ -57,7 +57,7 @@ func andOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
return crossFunction(d, context, expressionNode, performBoolOp(
func(b1 bool, b2 bool) bool {
return b1 && b2
}))
}), false)
}
func notOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {

View File

@@ -62,7 +62,7 @@ func sequenceFor(d *dataTreeNavigator, context Context, matchingNode *CandidateN
}
return &CandidateNode{Node: &node, Document: document, Path: path}, nil
})
}, false)
if err != nil {
return nil, err

View File

@@ -18,48 +18,23 @@ func deleteChildOperator(d *dataTreeNavigator, context Context, expressionNode *
for el := nodesToDelete.MatchingNodes.Back(); el != nil; el = el.Prev() {
candidate := el.Value.(*CandidateNode)
if len(candidate.Path) > 0 {
deleteImmediateChildOp := &Operation{
OperationType: deleteImmediateChildOpType,
Value: candidate.Path[len(candidate.Path)-1],
}
deleteImmediateChildOpNode := &ExpressionNode{
Operation: deleteImmediateChildOp,
Rhs: createTraversalTree(candidate.Path[0:len(candidate.Path)-1], traversePreferences{}, false),
}
_, err := d.GetMatchingNodes(contextToUse, deleteImmediateChildOpNode)
if err != nil {
return Context{}, err
}
//problem: context may already be '.a' and then I pass in '.a.a2'.
// should pass in .a2.
if candidate.Parent == nil {
log.Info("Could not find parent of %v", candidate.GetKey())
return context, nil
}
}
return context, nil
}
func deleteImmediateChildOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
parents, err := d.GetMatchingNodes(context, expressionNode.Rhs)
parentNode := candidate.Parent.Node
childPath := candidate.Path[len(candidate.Path)-1]
if err != nil {
return Context{}, err
}
childPath := expressionNode.Operation.Value
log.Debug("childPath to remove %v", childPath)
for el := parents.MatchingNodes.Front(); el != nil; el = el.Next() {
parent := el.Value.(*CandidateNode)
parentNode := unwrapDoc(parent.Node)
if parentNode.Kind == yaml.MappingNode {
deleteFromMap(parent, childPath)
deleteFromMap(candidate.Parent, childPath)
} else if parentNode.Kind == yaml.SequenceNode {
deleteFromArray(parent, childPath)
deleteFromArray(candidate.Parent, childPath)
} else {
return Context{}, fmt.Errorf("Cannot delete nodes from parent of tag %v", parentNode.Tag)
}
}
return context, nil
}

View File

@@ -21,6 +21,70 @@ var deleteOperatorScenarios = []expressionScenario{
"D0, P[], (doc)::{a: {a2: frood}}\n",
},
},
{
skipDoc: true,
document: `{a: {a1: fred, a2: frood}}`,
expression: `.a | del(.a1)`,
expected: []string{
"D0, P[a], (!!map)::{a2: frood}\n",
},
},
{
skipDoc: true,
document: `a: [1,2,3]`,
expression: `.a | del(.[1])`,
expected: []string{
"D0, P[a], (!!seq)::[1, 3]\n",
},
},
{
skipDoc: true,
document: `[0, {a: cat, b: dog}]`,
expression: `.[1] | del(.a)`,
expected: []string{
"D0, P[1], (!!map)::{b: dog}\n",
},
},
{
skipDoc: true,
document: `[{a: cat, b: dog}]`,
expression: `.[0] | del(.a)`,
expected: []string{
"D0, P[0], (!!map)::{b: dog}\n",
},
},
{
skipDoc: true,
document: `[{a: {b: thing, c: frog}}]`,
expression: `.[0].a | del(.b)`,
expected: []string{
"D0, P[0 a], (!!map)::{c: frog}\n",
},
},
{
skipDoc: true,
document: `[{a: {b: thing, c: frog}}]`,
expression: `.[0] | del(.a.b)`,
expected: []string{
"D0, P[0], (!!map)::{a: {c: frog}}\n",
},
},
{
skipDoc: true,
document: `{a: [0, {b: thing, c: frog}]}`,
expression: `.a[1] | del(.b)`,
expected: []string{
"D0, P[a 1], (!!map)::{c: frog}\n",
},
},
{
skipDoc: true,
document: `{a: [0, {b: thing, c: frog}]}`,
expression: `.a | del(.[1].b)`,
expected: []string{
"D0, P[a], (!!seq)::[0, {c: frog}]\n",
},
},
{
skipDoc: true,
document: `{a: {a1: fred, a2: frood}}`,

View File

@@ -4,7 +4,7 @@ import "gopkg.in/yaml.v3"
func equalsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- equalsOperation")
return crossFunction(d, context, expressionNode, isEquals(false))
return crossFunction(d, context, expressionNode, isEquals(false), false)
}
func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
@@ -29,5 +29,5 @@ func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *Candid
func notEqualsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- equalsOperation")
return crossFunction(d, context, expressionNode, isEquals(true))
return crossFunction(d, context, expressionNode, isEquals(true), false)
}

View File

@@ -21,6 +21,16 @@ var fileOperatorScenarios = []expressionScenario{
"D0, P[], (!!int)::0\n",
},
},
{
description: "Get file indices of multiple documents",
document: `{a: cat}`,
document2: `{a: cat}`,
expression: `fileIndex`,
expected: []string{
"D0, P[], (!!int)::0\n",
"D0, P[], (!!int)::1\n",
},
},
{
description: "Get file index alias",
document: `{a: cat}`,

View File

@@ -28,6 +28,15 @@ var hasOperatorScenarios = []expressionScenario{
"D0, P[3], (!!bool)::false\n",
},
},
{
description: "Select, checking for existence of deep paths",
subdescription: "Simply pipe in parent expressions into `has`",
document: "- {a: {b: {c: cat}}}\n- {a: {b: {d: dog}}}",
expression: `.[] | select(.a.b | has("c"))`,
expected: []string{
"D0, P[0], (!!map)::{a: {b: {c: cat}}}\n",
},
},
{
dontFormatInputForDoc: true,
description: "Has array index",

View File

@@ -10,13 +10,14 @@ import (
)
type multiplyPreferences struct {
AppendArrays bool
TraversePrefs traversePreferences
AppendArrays bool
DeepMergeArrays bool
TraversePrefs traversePreferences
}
func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- MultiplyOperator")
return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)))
return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)), false)
}
func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
@@ -30,6 +31,7 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
var newBlank = lhs.CreateChild(nil, &yaml.Node{})
var newThing, err = mergeObjects(d, context, newBlank, lhs, multiplyPreferences{})
if err != nil {
return nil, err
@@ -37,11 +39,31 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
return mergeObjects(d, context, newThing, rhs, preferences)
} else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" {
return multiplyIntegers(lhs, rhs)
} else if (lhs.Node.Tag == "!!int" || lhs.Node.Tag == "!!float") && (rhs.Node.Tag == "!!int" || rhs.Node.Tag == "!!float") {
return multiplyFloats(lhs, rhs)
}
return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
}
}
func multiplyFloats(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
target := lhs.CreateChild(nil, &yaml.Node{})
target.Node.Kind = yaml.ScalarNode
target.Node.Style = lhs.Node.Style
target.Node.Tag = "!!float"
lhsNum, err := strconv.ParseFloat(lhs.Node.Value, 64)
if err != nil {
return nil, err
}
rhsNum, err := strconv.ParseFloat(rhs.Node.Value, 64)
if err != nil {
return nil, err
}
target.Node.Value = fmt.Sprintf("%v", lhsNum*rhsNum)
return target, nil
}
func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
target := lhs.CreateChild(nil, &yaml.Node{})
target.Node.Kind = yaml.ScalarNode
@@ -98,11 +120,12 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
lhsPath := rhs.Path[pathIndexToStartFrom:]
assignmentOp := &Operation{OperationType: assignAttributesOpType}
if rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode {
if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
assignmentOp.OperationType = addAssignOpType
} else if !preferences.DeepMergeArrays && rhs.Node.Kind == yaml.SequenceNode ||
(rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode) {
assignmentOp.OperationType = assignOpType
assignmentOp.UpdateAssign = false
} else if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
assignmentOp.OperationType = addAssignOpType
}
rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}

View File

@@ -4,7 +4,57 @@ import (
"testing"
)
var doc1 = `list:
# Hi this is a comment.
# Hello this is another comment.
- "abc"`
var doc2 = `list2:
# This is yet another comment.
# Indeed this is yet another comment.
- "123"`
var docExpected = `D0, P[], (!!map)::list:
# Hi this is a comment.
# Hello this is another comment.
- "abc"
list2:
# This is yet another comment.
# Indeed this is yet another comment.
- "123"
`
var multiplyOperatorScenarios = []expressionScenario{
{
description: "Multiply integers",
expression: `3 * 4`,
expected: []string{
"D0, P[], (!!int)::12\n",
},
},
{
skipDoc: true,
document: doc1,
document2: doc2,
expression: `select(fi == 0) * select(fi == 1)`,
expected: []string{
docExpected,
},
},
{
skipDoc: true,
expression: `3 * 4.5`,
expected: []string{
"D0, P[], (!!float)::13.5\n",
},
},
{
skipDoc: true,
expression: `4.5 * 3`,
expected: []string{
"D0, P[], (!!float)::13.5\n",
},
},
{
skipDoc: true,
document: `{a: {also: [1]}, b: {also: me}}`,
@@ -157,7 +207,7 @@ var multiplyOperatorScenarios = []expressionScenario{
{
skipDoc: true,
document: `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,
expression: `.a *? .b`,
expression: `.a *?d .b`,
expected: []string{
"D0, P[a], (!!seq)::[{thing: two}]\n",
},
@@ -186,6 +236,15 @@ var multiplyOperatorScenarios = []expressionScenario{
"D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n",
},
},
{
description: "Merge, deeply merging arrays",
subdescription: "Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.",
document: `{a: [{name: fred, age: 12}, {name: bob, age: 32}], b: [{name: fred, age: 34}]}`,
expression: `.a *d .b`,
expected: []string{
"D0, P[a], (!!seq)::[{name: fred, age: 34}, {name: bob, age: 32}]\n",
},
},
{
description: "Merge to prefix an element",
document: `{a: cat, b: dog}`,
@@ -224,5 +283,5 @@ func TestMultiplyOperatorScenarios(t *testing.T) {
for _, tt := range multiplyOperatorScenarios {
testScenario(t, &tt)
}
documentScenarios(t, "Multiply", multiplyOperatorScenarios)
documentScenarios(t, "Multiply (Merge)", multiplyOperatorScenarios)
}

View File

@@ -6,17 +6,35 @@ import (
var reduceOperatorScenarios = []expressionScenario{
{
document: `[10,2, 5, 3]`,
expression: `.[] as $item ireduce (0; . + $item)`,
description: "Sum numbers",
document: `[10,2, 5, 3]`,
expression: `.[] as $item ireduce (0; . + $item)`,
expected: []string{
"D0, P[], (!!int)::20\n",
},
},
{
description: "Merge all yaml files together",
document: `a: cat`,
document2: `b: dog`,
expression: `. as $item ireduce ({}; . * $item )`,
expected: []string{
"D0, P[], (!!map)::a: cat\nb: dog\n",
},
},
{
description: "Convert an array to an object",
document: `[{name: Cathy, has: apples},{name: Bob, has: bananas}]`,
expression: `.[] as $item ireduce ({}; .[$item | .name] = ($item | .has) )`,
expected: []string{
"D0, P[], (!!map)::Cathy: apples\nBob: bananas\n",
},
},
}
func TestReduceOperatorScenarios(t *testing.T) {
for _, tt := range reduceOperatorScenarios {
testScenario(t, &tt)
}
// documentScenarios(t, "Reduce", reduceOperatorScenarios)
documentScenarios(t, "Reduce", reduceOperatorScenarios)
}

View File

@@ -3,11 +3,77 @@ package yqlib
import (
"container/list"
"fmt"
"regexp"
"strings"
"gopkg.in/yaml.v3"
)
func getSubstituteParameters(d *dataTreeNavigator, block *ExpressionNode, context Context) (string, string, error) {
regEx := ""
replacementText := ""
regExNodes, err := d.GetMatchingNodes(context, block.Lhs)
if err != nil {
return "", "", err
}
if regExNodes.MatchingNodes.Front() != nil {
regEx = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
}
log.Debug("regEx %v", regEx)
replacementNodes, err := d.GetMatchingNodes(context, block.Rhs)
if err != nil {
return "", "", err
}
if replacementNodes.MatchingNodes.Front() != nil {
replacementText = replacementNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
}
return regEx, replacementText, nil
}
func substitute(original string, regex *regexp.Regexp, replacement string) *yaml.Node {
replacedString := regex.ReplaceAllString(original, replacement)
return &yaml.Node{Kind: yaml.ScalarNode, Value: replacedString, Tag: "!!str"}
}
func substituteStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
//rhs block operator
//lhs of block = regex
//rhs of block = replacement expression
block := expressionNode.Rhs
regExStr, replacementText, err := getSubstituteParameters(d, block, context)
if err != nil {
return Context{}, err
}
regEx, err := regexp.Compile(regExStr)
if err != nil {
return Context{}, err
}
var results = list.New()
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
node := unwrapDoc(candidate.Node)
if node.Tag != "!!str" {
return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation.", node.Tag)
}
targetNode := substitute(node.Value, regEx, replacementText)
result := candidate.CreateChild(nil, targetNode)
results.PushBack(result)
}
return context.ChildContext(results), nil
}
func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- joinStringOperator")
joinStr := ""
@@ -26,7 +92,7 @@ func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *E
candidate := el.Value.(*CandidateNode)
node := unwrapDoc(candidate.Node)
if node.Kind != yaml.SequenceNode {
return Context{}, fmt.Errorf("Cannot join with %v, can only join arrays of scalars", node.Tag)
return Context{}, fmt.Errorf("cannot join with %v, can only join arrays of scalars", node.Tag)
}
targetNode := join(node.Content, joinStr)
result := candidate.CreateChild(nil, targetNode)

View File

@@ -13,6 +13,24 @@ var stringsOperatorScenarios = []expressionScenario{
"D0, P[], (!!str)::cat; meow; 1; ; true\n",
},
},
{
description: "Substitute / Replace string",
subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)\nNote the use of `|=` to run in context of the current string value.",
document: `a: dogs are great`,
expression: `.a |= sub("dogs", "cats")`,
expected: []string{
"D0, P[], (doc)::a: cats are great\n",
},
},
{
description: "Substitute / Replace string with regex",
subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)\nNote the use of `|=` to run in context of the current string value.",
document: "a: cat\nb: heat",
expression: `.[] |= sub("(a)", "${1}r")`,
expected: []string{
"D0, P[], (doc)::a: cart\nb: heart\n",
},
},
{
description: "Split strings",
document: `"cat; meow; 1; ; true"`,

View File

@@ -0,0 +1,97 @@
package yqlib
import (
"fmt"
"strconv"
yaml "gopkg.in/yaml.v3"
)
func createSubtractOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {
return &ExpressionNode{Operation: &Operation{OperationType: subtractOpType},
Lhs: lhs,
Rhs: rhs}
}
func subtractAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
assignmentOp := &Operation{OperationType: assignOpType}
assignmentOp.UpdateAssign = true
selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createSubtractOp(selfExpression, expressionNode.Rhs)}
return d.GetMatchingNodes(context, assignmentOpNode)
}
func subtractOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("Subtract operator")
return crossFunction(d, context, expressionNode, subtract, false)
}
func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
lhs.Node = unwrapDoc(lhs.Node)
rhs.Node = unwrapDoc(rhs.Node)
lhsNode := lhs.Node
if lhsNode.Tag == "!!null" {
return lhs.CreateChild(nil, rhs.Node), nil
}
target := lhs.CreateChild(nil, &yaml.Node{})
switch lhsNode.Kind {
case yaml.MappingNode:
return nil, fmt.Errorf("Maps not yet supported for subtraction")
case yaml.SequenceNode:
return nil, fmt.Errorf("Sequences not yet supported for subtraction")
// target.Node.Kind = yaml.SequenceNode
// target.Node.Style = lhsNode.Style
// target.Node.Tag = "!!seq"
// target.Node.Content = append(lhsNode.Content, toNodes(rhs)...)
case yaml.ScalarNode:
if rhs.Node.Kind != yaml.ScalarNode {
return nil, fmt.Errorf("%v (%v) cannot be added to a %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag)
}
target.Node.Kind = yaml.ScalarNode
target.Node.Style = lhsNode.Style
return subtractScalars(target, lhsNode, rhs.Node)
}
return target, nil
}
func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*CandidateNode, error) {
if lhs.Tag == "!!str" {
return nil, fmt.Errorf("strings cannot be subtracted")
} else if lhs.Tag == "!!int" && rhs.Tag == "!!int" {
lhsNum, err := strconv.Atoi(lhs.Value)
if err != nil {
return nil, err
}
rhsNum, err := strconv.Atoi(rhs.Value)
if err != nil {
return nil, err
}
result := lhsNum - rhsNum
target.Node.Tag = "!!int"
target.Node.Value = fmt.Sprintf("%v", result)
} else if (lhs.Tag == "!!int" || lhs.Tag == "!!float") && (rhs.Tag == "!!int" || rhs.Tag == "!!float") {
lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
if err != nil {
return nil, err
}
rhsNum, err := strconv.ParseFloat(rhs.Value, 64)
if err != nil {
return nil, err
}
result := lhsNum - rhsNum
target.Node.Tag = "!!float"
target.Node.Value = fmt.Sprintf("%v", result)
} else {
return nil, fmt.Errorf("%v cannot be added to %v", lhs.Tag, rhs.Tag)
}
return target, nil
}

View File

@@ -0,0 +1,50 @@
package yqlib
import (
"testing"
)
var subtractOperatorScenarios = []expressionScenario{
{
description: "Number subtraction - float",
subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.",
document: `{a: 3, b: 4.5}`,
expression: `.a = .a - .b`,
expected: []string{
"D0, P[], (doc)::{a: -1.5, b: 4.5}\n",
},
},
{
description: "Number subtraction - float",
subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.",
document: `{a: 3, b: 4.5}`,
expression: `.a = .a - .b`,
expected: []string{
"D0, P[], (doc)::{a: -1.5, b: 4.5}\n",
},
},
{
description: "Number subtraction - int",
subdescription: "If both the lhs and rhs are ints then the expression will be calculated with ints.",
document: `{a: 3, b: 4}`,
expression: `.a = .a - .b`,
expected: []string{
"D0, P[], (doc)::{a: -1, b: 4}\n",
},
},
{
description: "Decrement numbers",
document: `{a: 3, b: 5}`,
expression: `.[] -= 1`,
expected: []string{
"D0, P[], (doc)::{a: 2, b: 4}\n",
},
},
}
func TestSubtractOperatorScenarios(t *testing.T) {
for _, tt := range subtractOperatorScenarios {
testScenario(t, &tt)
}
documentScenarios(t, "Subtract", subtractOperatorScenarios)
}

View File

@@ -24,7 +24,20 @@ func emptyOperator(d *dataTreeNavigator, context Context, expressionNode *Expres
type crossFunctionCalculation func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)
func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (Context, error) {
func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *CandidateNode, rhs Context, calculation crossFunctionCalculation, results *list.List) error {
for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() {
log.Debugf("Applying calc")
rhsCandidate := rightEl.Value.(*CandidateNode)
resultCandidate, err := calculation(d, context, lhsCandidate, rhsCandidate)
if err != nil {
return err
}
results.PushBack(resultCandidate)
}
return nil
}
func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation, calcWhenEmpty bool) (Context, error) {
var results = list.New()
lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
if err != nil {
@@ -38,24 +51,33 @@ func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *Expressi
return Context{}, err
}
for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
lhsCandidate := el.Value.(*CandidateNode)
for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() {
log.Debugf("Applying calc")
rhsCandidate := rightEl.Value.(*CandidateNode)
resultCandidate, err := calculation(d, context, lhsCandidate, rhsCandidate)
if calcWhenEmpty && lhs.MatchingNodes.Len() == 0 {
if rhs.MatchingNodes.Len() == 0 {
resultCandidate, err := calculation(d, context, nil, nil)
if err != nil {
return Context{}, err
}
results.PushBack(resultCandidate)
}
err := resultsForRhs(d, context, nil, rhs, calculation, results)
if err != nil {
return Context{}, err
}
}
for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
lhsCandidate := el.Value.(*CandidateNode)
err := resultsForRhs(d, context, lhsCandidate, rhs, calculation, results)
if err != nil {
return Context{}, err
}
}
return context.ChildContext(results), nil
}
func crossFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (Context, error) {
func crossFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation, calcWhenEmpty bool) (Context, error) {
var results = list.New()
var evaluateAllTogether = true
@@ -66,11 +88,11 @@ func crossFunction(d *dataTreeNavigator, context Context, expressionNode *Expres
}
}
if evaluateAllTogether {
return doCrossFunc(d, context, expressionNode, calculation)
return doCrossFunc(d, context, expressionNode, calculation, calcWhenEmpty)
}
for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
innerResults, err := doCrossFunc(d, context.SingleChildContext(matchEl.Value.(*CandidateNode)), expressionNode, calculation)
innerResults, err := doCrossFunc(d, context.SingleChildContext(matchEl.Value.(*CandidateNode)), expressionNode, calculation, calcWhenEmpty)
if err != nil {
return Context{}, err
}

View File

@@ -34,7 +34,7 @@ func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEn
unwrapScalar: unwrapScalar,
colorsEnabled: colorsEnabled,
indent: indent,
printDocSeparators: printDocSeparators,
printDocSeparators: !outputToJSON && printDocSeparators,
firstTimePrinting: true,
treeNavigator: NewDataTreeNavigator(),
}

View File

@@ -130,7 +130,9 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
func TestPrinterMultipleDocsJson(t *testing.T) {
var output bytes.Buffer
var writer = bufio.NewWriter(&output)
printer := NewPrinter(writer, true, true, false, 0, false)
// note printDocSeparators is true, it should still not print document separators
// when outputing JSON.
printer := NewPrinter(writer, true, true, false, 0, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
if err != nil {

View File

@@ -2,6 +2,7 @@
- increment version in snapcraft.yaml
- increment version in github-action/Dockerfile
- make sure local build passes
- run ./scripts/secure.sh (manual because docker platforms)
- commit version update changes
- tag git with same version number
- commit vX tag - this will trigger github actions

View File

@@ -10,12 +10,3 @@ else
./bin/golangci-lint run --timeout=5m
fi
# ./bin/golangci-lint \
# --tests \
# --vendor \
# --disable=aligncheck \
# --disable=gotype \
# --disable=goconst \
# --disable=gocyclo \
# --deadline=300s \
# ./...

View File

@@ -1,4 +1,5 @@
#!/bin/sh
set -ex
go get golang.org/x/tools/cmd/goimports
wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.24.0
wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.37.1
# wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.6.1

11
scripts/secure.sh Executable file
View File

@@ -0,0 +1,11 @@
#!/bin/bash
set -o errexit
set -o pipefail
if command -v gosec &> /dev/null
then
gosec ${PWD}
else
./bin/gosec ${PWD}
fi

View File

@@ -1,5 +1,5 @@
name: yq
version: '4.5.1'
version: '4.7.1'
summary: A lightweight and portable command-line YAML processor
description: |
The aim of the project is to be the jq or sed of yaml files.