1
0
mirror of https://github.com/taigrr/yq synced 2025-01-18 04:53:17 -08:00

Compare commits

..

22 Commits

Author SHA1 Message Date
Mike Farah
6c26344449 Incrementing version 2021-04-15 16:22:24 +10:00
Mike Farah
daf0bfe1b9 Added string substitute command 2021-04-15 16:09:47 +10:00
Mike Farah
750a00ec35 Added "expand" to explode docs for searchability 2021-04-13 13:59:26 +10:00
Mike Farah
25e0a824c5 Fixed alternative operator when LHS has empty matches 2021-04-13 10:53:46 +10:00
Mike Farah
095f921f62 4.6.3 release 2021-03-25 08:45:59 +11:00
Mike Farah
12d3425b4a Added subtract operator (numbers only) 2021-03-25 08:12:01 +11:00
Mike Farah
0249f00bd5 Added chocolate badges thanks @adriens 2021-03-23 20:40:39 +11:00
Mike Farah
42b1bf9678 updated release notes to run gosec manually 2021-03-19 13:41:56 +11:00
Mike Farah
c632fd3641 cannot run gosec on all docker platforms, removing from devtools 2021-03-19 13:40:24 +11:00
Mike Farah
0470e5cd69 cannot run gosec on all docker platforms, removing for now 2021-03-19 13:21:07 +11:00
Mike Farah
35c6d07248 Bump version 2021-03-19 12:59:00 +11:00
Mike Farah
3f0be107d4 Bump dependencies 2021-03-19 12:58:43 +11:00
Mike Farah
21a9e506cb Fixed merge comments 2021-03-19 12:54:03 +11:00
Mike Farah
3722367fbb Dont print doc separators for JSON (https://github.com/mikefarah/yq/issues/735) 2021-03-19 12:40:56 +11:00
Mike Farah
f7b50e9853 Fixed += operator (https://github.com/mikefarah/yq/issues/750) 2021-03-19 12:36:05 +11:00
Mike Farah
4f3fe256aa Fixed precedence of CREATE_MAP (https://github.com/mikefarah/yq/issues/753) 2021-03-19 12:09:32 +11:00
Mike Farah
2595a929c9 Fixing doc links 2021-03-19 11:52:13 +11:00
Mike Farah
b2186d5404 Added gosec 2021-03-03 19:44:34 +11:00
Mike Farah
e93c43f7a0 Improving docs 2021-02-26 11:31:43 +11:00
Mike Farah
b1a5387cdd Update readme 2021-02-25 17:04:08 +11:00
Mike Farah
5911ab2929 Bump version 2021-02-25 16:52:49 +11:00
Mike Farah
2ed5b2ff59 Improved lexer performance! 2021-02-25 16:47:55 +11:00
48 changed files with 615 additions and 145 deletions

View File

@@ -1 +1 @@
bin
bin/*

View File

@@ -6,26 +6,5 @@ RUN set -e -x \
&& /opt/devtools.sh
ENV PATH=/go/bin:$PATH
# install mkdocs
RUN set -ex \
&& buildDeps=' \
build-essential \
python3-dev \
' \
&& apt-get update && apt-get install -y --no-install-recommends \
$buildDeps \
python3 \
python3-setuptools \
python3-wheel \
python3-pip \
&& pip3 install --upgrade \
pip \
'Markdown>=2.6.9' \
'mkdocs>=0.16.3' \
'mkdocs-material>=1.10.1' \
'markdown-include>=0.5.1' \
&& apt-get purge -y --auto-remove $buildDeps \
&& rm -rf /var/lib/apt/lists/*
ENV CGO_ENABLED 0
ENV GOPATH /go:/yq

View File

@@ -17,6 +17,7 @@ help:
@echo ' make vendor Install dependencies to vendor directory.'
@echo ' make format Run code formatter.'
@echo ' make check Run static code analysis (lint).'
@echo ' make secure Run gosec.'
@echo ' make test Run tests on project.'
@echo ' make cover Run tests and capture code coverage metrics on project.'
@echo ' make clean Clean the directory tree of produced artifacts.'
@@ -84,6 +85,10 @@ format: vendor
check: format
${DOCKRUN} bash ./scripts/check.sh
.PHONY: secure
secure:
${DOCKRUN} bash ./scripts/secure.sh
.PHONY: test
test: check
${DOCKRUN} bash ./scripts/test.sh
@@ -96,11 +101,6 @@ cover: check
@find cover -type d -exec chmod 755 {} \; || :
@find cover -type f -exec chmod 644 {} \; || :
.PHONY: build-docs
build-docs: prepare mkdocs.yml mkdocs/*
${DOCKRUN} mkdocs build
@find docs -type d -exec chmod 755 {} \; || :
@find docs -type f -exec chmod 644 {} \; || :
.PHONY: release
release: xcompile

View File

@@ -119,6 +119,8 @@ See [webi](https://webinstall.dev/)
Supported by @adithyasunil26 (https://github.com/webinstall/webi-installers/tree/master/yq)
### Windows:
[![Chocolatey](https://img.shields.io/chocolatey/v/yq.svg)](https://chocolatey.org/packages/yq)
[![Chocolatey](https://img.shields.io/chocolatey/dt/yq.svg)](https://chocolatey.org/packages/yq)
```
choco install yq
```
@@ -151,19 +153,21 @@ sudo apt install yq -y
Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
## Features
- [Detailed documentation with many examples](https://mikefarah.gitbook.io/yq/)
- Written in portable go, so you can download a lovely dependency free binary
- Uses similar syntax as `jq` but works with YAML and JSON files
- Fully supports multi document yaml files
- Colorized yaml output
- [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/v/v4.x/traverse)
- [Sort yaml by keys](https://mikefarah.gitbook.io/yq/v/v4.x/sort-keys)
- Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/comment-operators), [styling](https://mikefarah.gitbook.io/yq/style), [tags](https://mikefarah.gitbook.io/yq/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/anchor-and-alias-operators).
- [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/operators/traverse-read)
- [Sort yaml by keys](https://mikefarah.gitbook.io/yq/operators/sort-keys)
- Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/operators/comment-operators), [styling](https://mikefarah.gitbook.io/yq/operators/style), [tags](https://mikefarah.gitbook.io/yq/operators/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators).
- [Update yaml inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
- [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/v/v4.x/select#select-and-update-matching-values-in-map)
- [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map)
- Keeps yaml formatting and comments when updating (though there are issues with whitespace)
- [Convert to/from json to yaml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
- [Pipe data in by using '-'](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate)
- [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
- [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things.
## [Usage](https://mikefarah.gitbook.io/yq/)

View File

@@ -11,7 +11,7 @@ var (
GitDescribe string
// Version is main version number that is being run at the moment.
Version = "4.6.0"
Version = "4.7.0"
// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release

View File

@@ -1,4 +1,4 @@
FROM mikefarah/yq:4.6.0
FROM mikefarah/yq:4.7.0
COPY entrypoint.sh /entrypoint.sh

8
go.mod
View File

@@ -3,12 +3,12 @@ module github.com/mikefarah/yq/v4
require (
github.com/elliotchance/orderedmap v1.4.0
github.com/fatih/color v1.10.0
github.com/goccy/go-yaml v1.8.8
github.com/jinzhu/copier v0.2.3
github.com/spf13/cobra v1.1.1
github.com/goccy/go-yaml v1.8.9
github.com/jinzhu/copier v0.2.8
github.com/spf13/cobra v1.1.3
github.com/timtadh/data-structures v0.5.3 // indirect
github.com/timtadh/lexmachine v0.2.2
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e // indirect
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)

26
go.sum
View File

@@ -52,8 +52,8 @@ github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTM
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/goccy/go-yaml v1.8.8 h1:MGfRB1GeSn/hWXYWS2Pt67iC2GJNnebdIro01ddyucA=
github.com/goccy/go-yaml v1.8.8/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
github.com/goccy/go-yaml v1.8.9 h1:4AEXg2qx+/w29jXnXpMY6mTckmYu1TMoHteKuMf0HFg=
github.com/goccy/go-yaml v1.8.9/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -101,8 +101,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jinzhu/copier v0.2.3 h1:Oe09ju+9qft7TffZ7l/04AB2f8u1+V4ZMxmp/nnqeOs=
github.com/jinzhu/copier v0.2.3/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
github.com/jinzhu/copier v0.2.8 h1:N8MbL5niMwE3P4dOwurJixz5rMkKfujmMRFmAanSzWE=
github.com/jinzhu/copier v0.2.8/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -168,10 +168,9 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@@ -180,7 +179,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -257,10 +255,9 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e h1:XNp2Flc/1eWQGk5BLzqTAN7fQIwIbfyVTuVxXxZh73M=
golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -270,7 +267,6 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -283,7 +279,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
@@ -310,7 +305,6 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -321,11 +315,9 @@ gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -86,7 +86,14 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) {
if n.Node.Style == 0 {
n.Node.Style = other.Node.Style
}
n.Node.FootComment = n.Node.FootComment + other.Node.FootComment
n.Node.HeadComment = n.Node.HeadComment + other.Node.HeadComment
n.Node.LineComment = n.Node.LineComment + other.Node.LineComment
if other.Node.FootComment != "" {
n.Node.FootComment = other.Node.FootComment
}
if other.Node.HeadComment != "" {
n.Node.HeadComment = other.Node.HeadComment
}
if other.Node.LineComment != "" {
n.Node.LineComment = other.Node.LineComment
}
}

View File

@@ -101,7 +101,7 @@ will output
- hello
```
## Update array (append)
## Append to array
Given a sample.yml file of:
```yaml
a:
@@ -127,6 +127,36 @@ b:
- 4
```
## Relative append
Given a sample.yml file of:
```yaml
a:
a1:
b:
- cat
a2:
b:
- dog
a3: {}
```
then
```bash
yq eval '.a[].b += ["mouse"]' sample.yml
```
will output
```yaml
a:
a1:
b:
- cat
- mouse
a2:
b:
- dog
- mouse
a3: {b: [mouse]}
```
## String concatenation
Given a sample.yml file of:
```yaml
@@ -143,22 +173,6 @@ a: catmeow
b: meow
```
## Relative string concatenation
Given a sample.yml file of:
```yaml
a: cat
b: meow
```
then
```bash
yq eval '.a += .b' sample.yml
```
will output
```yaml
a: catmeow
b: meow
```
## Number addition - float
If the lhs or rhs are floats then the expression will be calculated with floats.
@@ -195,18 +209,20 @@ a: 7
b: 4
```
## Increment number
## Increment numbers
Given a sample.yml file of:
```yaml
a: 3
b: 5
```
then
```bash
yq eval '.a += 1' sample.yml
yq eval '.[] += 1' sample.yml
```
will output
```yaml
a: 4
b: 6
```
## Add to null

View File

@@ -1,4 +1,4 @@
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference aliases and remove anchor names).
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.

View File

@@ -1,10 +1,4 @@
This operator is used to update node values. It can be used in either the:
### plain form: `=`
Which will assign the LHS node values to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.
### relative form: `|=`
This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment.
## Create yaml file
Running
```bash

View File

@@ -1,4 +1,15 @@
Use these comment operators to set or retrieve comments.
Like the `=` and `|=` assign operators, the same syntax applies when updating comments:
### plain form: `=`
This will assign the LHS nodes comments to the expression on the RHS. The RHS is run against the matching nodes in the pipeline
### relative form: `|=`
Similar to the plain form, however the RHS evaluates against each matching LHS node! This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.
## Set line comment
Given a sample.yml file of:
```yaml

View File

@@ -19,6 +19,29 @@ true
false
```
## Select, checking for existence of deep paths
Simply pipe in parent expressions into `has`
Given a sample.yml file of:
```yaml
- a:
b:
c: cat
- a:
b:
d: dog
```
then
```bash
yq eval '.[] | select(.a.b | has("c"))' sample.yml
```
will output
```yaml
a:
b:
c: cat
```
## Has array index
Given a sample.yml file of:
```yaml

View File

@@ -18,6 +18,40 @@ will output
cat; meow; 1; ; true
```
## Substitute / Replace string
This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
Given a sample.yml file of:
```yaml
a: dogs are great
```
then
```bash
yq eval '.a |= sub("dogs", "cats")' sample.yml
```
will output
```yaml
a: cats are great
```
## Substitute / Replace string with regex
This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
Given a sample.yml file of:
```yaml
a: cat
b: heat
```
then
```bash
yq eval '.[] |= sub("([a])", "${1}r")' sample.yml
```
will output
```yaml
a: cart
b: heart
```
## Split strings
Given a sample.yml file of:
```yaml

71
pkg/yqlib/doc/Subtract.md Normal file
View File

@@ -0,0 +1,71 @@
## Number subtraction - float
If the lhs or rhs are floats then the expression will be calculated with floats.
Given a sample.yml file of:
```yaml
a: 3
b: 4.5
```
then
```bash
yq eval '.a = .a - .b' sample.yml
```
will output
```yaml
a: -1.5
b: 4.5
```
## Number subtraction - float
If the lhs or rhs are floats then the expression will be calculated with floats.
Given a sample.yml file of:
```yaml
a: 3
b: 4.5
```
then
```bash
yq eval '.a = .a - .b' sample.yml
```
will output
```yaml
a: -1.5
b: 4.5
```
## Number subtraction - int
If both the lhs and rhs are ints then the expression will be calculated with ints.
Given a sample.yml file of:
```yaml
a: 3
b: 4
```
then
```bash
yq eval '.a = .a - .b' sample.yml
```
will output
```yaml
a: -1
b: 4
```
## Decrement numbers
Given a sample.yml file of:
```yaml
a: 3
b: 5
```
then
```bash
yq eval '.[] -= 1' sample.yml
```
will output
```yaml
a: 2
b: 4
```

View File

@@ -1,4 +1,4 @@
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference aliases and remove anchor names).
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.

View File

@@ -1 +1,11 @@
Use these comment operators to set or retrieve comments.
Use these comment operators to set or retrieve comments.
Like the `=` and `|=` assign operators, the same syntax applies when updating comments:
### plain form: `=`
This will assign the LHS nodes comments to the expression on the RHS. The RHS is run against the matching nodes in the pipeline
### relative form: `|=`
Similar to the plain form, however the RHS evaluates against each matching LHS node! This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.

View File

@@ -5,9 +5,6 @@ import (
"strings"
)
var myPathTokeniser = newExpressionTokeniser()
var myPathPostfixer = newExpressionPostFixer()
type ExpressionNode struct {
Operation *Operation
Lhs *ExpressionNode
@@ -19,19 +16,21 @@ type ExpressionParser interface {
}
type expressionParserImpl struct {
pathTokeniser expressionTokeniser
pathPostFixer expressionPostFixer
}
func NewExpressionParser() ExpressionParser {
return &expressionParserImpl{}
return &expressionParserImpl{newExpressionTokeniser(), newExpressionPostFixer()}
}
func (p *expressionParserImpl) ParseExpression(expression string) (*ExpressionNode, error) {
tokens, err := myPathTokeniser.Tokenise(expression)
tokens, err := p.pathTokeniser.Tokenise(expression)
if err != nil {
return nil, err
}
var Operations []*Operation
Operations, err = myPathPostfixer.ConvertToPostfix(tokens)
Operations, err = p.pathPostFixer.ConvertToPostfix(tokens)
if err != nil {
return nil, err
}

View File

@@ -12,6 +12,21 @@ var pathTests = []struct {
expectedTokens []interface{}
expectedPostFix []interface{}
}{
{
`[]|join(".")`,
append(make([]interface{}, 0), "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")"),
append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE", ". (string)", "JOIN", "PIPE"),
},
{
`{"cool": .b or .c}`,
append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "b", "OR", "c", "}"),
append(make([]interface{}, 0), "cool (string)", "b", "c", "OR", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
},
{
`{"cool": []|join(".")}`,
append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")", "}"),
append(make([]interface{}, 0), "cool (string)", "EMPTY", "COLLECT", "SHORT_PIPE", ". (string)", "JOIN", "PIPE", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
},
{
`.a as $item ireduce (0; . + $item)`, // note - add code to shuffle reduce to this position for postfix
append(make([]interface{}, 0), "a", "ASSIGN_VARIABLE", "GET_VARIABLE", "REDUCE", "(", "0 (int64)", "BLOCK", "SELF", "ADD", "GET_VARIABLE", ")"),

View File

@@ -264,6 +264,8 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`splitDoc`), opToken(splitDocumentOpType))
lexer.Add([]byte(`join`), opToken(joinStringOpType))
lexer.Add([]byte(`sub`), opToken(subStringOpType))
lexer.Add([]byte(`split`), opToken(splitStringOpType))
lexer.Add([]byte(`keys`), opToken(keysOpType))
@@ -325,10 +327,12 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`\*[\+|\?d]*`), multiplyWithPrefs())
lexer.Add([]byte(`\+`), opToken(addOpType))
lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
lexer.Add([]byte(`\-`), opToken(subtractOpType))
lexer.Add([]byte(`\-=`), opToken(subtractAssignOpType))
lexer.Add([]byte(`\$[a-zA-Z_-0-9]+`), getVariableOpToken())
lexer.Add([]byte(`as`), opToken(assignVariableOpType))
err := lexer.Compile()
err := lexer.CompileNFA()
if err != nil {
return nil, err
}

View File

@@ -35,6 +35,7 @@ var pipeOpType = &operationType{Type: "PIPE", NumArgs: 2, Precedence: 30, Handle
var assignOpType = &operationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: assignUpdateOperator}
var addAssignOpType = &operationType{Type: "ADD_ASSIGN", NumArgs: 2, Precedence: 40, Handler: addAssignOperator}
var subtractAssignOpType = &operationType{Type: "SUBTRACT_ASSIGN", NumArgs: 2, Precedence: 40, Handler: subtractAssignOperator}
var assignAttributesOpType = &operationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: assignAttributesOperator}
var assignStyleOpType = &operationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: assignStyleOperator}
@@ -46,11 +47,14 @@ var assignAliasOpType = &operationType{Type: "ASSIGN_ALIAS", NumArgs: 2, Precede
var multiplyOpType = &operationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 42, Handler: multiplyOperator}
var addOpType = &operationType{Type: "ADD", NumArgs: 2, Precedence: 42, Handler: addOperator}
var subtractOpType = &operationType{Type: "SUBTRACT", NumArgs: 2, Precedence: 42, Handler: subtractOperator}
var alternativeOpType = &operationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 42, Handler: alternativeOperator}
var equalsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: equalsOperator}
var notEqualsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: notEqualsOperator}
var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 40, Handler: createMapOperator}
//createmap needs to be above union, as we use union to build the components of the objects
var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 15, Handler: createMapOperator}
var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: 45, Handler: pipeOperator}
@@ -71,6 +75,7 @@ var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50,
var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator}
var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}
var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator}
var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}

View File

@@ -16,9 +16,9 @@ func createAddOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {
func addAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
assignmentOp := &Operation{OperationType: assignOpType}
assignmentOp.UpdateAssign = false
assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createAddOp(expressionNode.Lhs, expressionNode.Rhs)}
assignmentOp.UpdateAssign = true
selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createAddOp(selfExpression, expressionNode.Rhs)}
return d.GetMatchingNodes(context, assignmentOpNode)
}
@@ -39,7 +39,7 @@ func toNodes(candidate *CandidateNode) []*yaml.Node {
func addOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("Add operator")
return crossFunction(d, context, expressionNode, add)
return crossFunction(d, context, expressionNode, add, false)
}
func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

View File

@@ -63,7 +63,7 @@ var addOperatorScenarios = []expressionScenario{
},
},
{
description: "Update array (append)",
description: "Append to array",
document: `{a: [1,2], b: [3,4]}`,
expression: `.a = .a + .b`,
expected: []string{
@@ -71,17 +71,17 @@ var addOperatorScenarios = []expressionScenario{
},
},
{
description: "String concatenation",
document: `{a: cat, b: meow}`,
expression: `.a = .a + .b`,
description: "Relative append",
document: `a: { a1: {b: [cat]}, a2: {b: [dog]}, a3: {} }`,
expression: `.a[].b += ["mouse"]`,
expected: []string{
"D0, P[], (doc)::{a: catmeow, b: meow}\n",
"D0, P[], (doc)::a: {a1: {b: [cat, mouse]}, a2: {b: [dog, mouse]}, a3: {b: [mouse]}}\n",
},
},
{
description: "Relative string concatenation",
description: "String concatenation",
document: `{a: cat, b: meow}`,
expression: `.a += .b`,
expression: `.a = .a + .b`,
expected: []string{
"D0, P[], (doc)::{a: catmeow, b: meow}\n",
},
@@ -105,11 +105,11 @@ var addOperatorScenarios = []expressionScenario{
},
},
{
description: "Increment number",
document: `{a: 3}`,
expression: `.a += 1`,
description: "Increment numbers",
document: `{a: 3, b: 5}`,
expression: `.[] += 1`,
expected: []string{
"D0, P[], (doc)::{a: 4}\n",
"D0, P[], (doc)::{a: 4, b: 6}\n",
},
},
{

View File

@@ -1,14 +1,14 @@
package yqlib
// corssFunction no matches
// can boolean use crossfunction
func alternativeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- alternative")
return crossFunction(d, context, expressionNode, alternativeFunc)
return crossFunction(d, context, expressionNode, alternativeFunc, true)
}
func alternativeFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
if lhs == nil {
return rhs, nil
}
lhs.Node = unwrapDoc(lhs.Node)
rhs.Node = unwrapDoc(rhs.Node)
log.Debugf("Alternative LHS: %v", lhs.Node.Tag)

View File

@@ -13,6 +13,14 @@ var alternativeOperatorScenarios = []expressionScenario{
"D0, P[a], (!!str)::bridge\n",
},
},
{
expression: `select(tag == "seq") // "cat"`,
skipDoc: true,
document: `a: frog`,
expected: []string{
"D0, P[], (!!str)::cat\n",
},
},
{
description: "LHS is not defined",
expression: `.a // "hello"`,

View File

@@ -143,5 +143,5 @@ func TestAssignOperatorScenarios(t *testing.T) {
for _, tt := range assignOperatorScenarios {
testScenario(t, &tt)
}
documentScenarios(t, "Assign", assignOperatorScenarios)
documentScenarios(t, "Assign (Update)", assignOperatorScenarios)
}

View File

@@ -49,7 +49,7 @@ func orOperator(d *dataTreeNavigator, context Context, expressionNode *Expressio
return crossFunction(d, context, expressionNode, performBoolOp(
func(b1 bool, b2 bool) bool {
return b1 || b2
}))
}), false)
}
func andOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@@ -57,7 +57,7 @@ func andOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
return crossFunction(d, context, expressionNode, performBoolOp(
func(b1 bool, b2 bool) bool {
return b1 && b2
}))
}), false)
}
func notOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {

View File

@@ -62,7 +62,7 @@ func sequenceFor(d *dataTreeNavigator, context Context, matchingNode *CandidateN
}
return &CandidateNode{Node: &node, Document: document, Path: path}, nil
})
}, false)
if err != nil {
return nil, err

View File

@@ -4,7 +4,7 @@ import "gopkg.in/yaml.v3"
func equalsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- equalsOperation")
return crossFunction(d, context, expressionNode, isEquals(false))
return crossFunction(d, context, expressionNode, isEquals(false), false)
}
func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
@@ -29,5 +29,5 @@ func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *Candid
func notEqualsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- equalsOperation")
return crossFunction(d, context, expressionNode, isEquals(true))
return crossFunction(d, context, expressionNode, isEquals(true), false)
}

View File

@@ -28,6 +28,15 @@ var hasOperatorScenarios = []expressionScenario{
"D0, P[3], (!!bool)::false\n",
},
},
{
description: "Select, checking for existence of deep paths",
subdescription: "Simply pipe in parent expressions into `has`",
document: "- {a: {b: {c: cat}}}\n- {a: {b: {d: dog}}}",
expression: `.[] | select(.a.b | has("c"))`,
expected: []string{
"D0, P[0], (!!map)::{a: {b: {c: cat}}}\n",
},
},
{
dontFormatInputForDoc: true,
description: "Has array index",

View File

@@ -17,7 +17,7 @@ type multiplyPreferences struct {
func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- MultiplyOperator")
return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)))
return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)), false)
}
func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
@@ -31,6 +31,7 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
var newBlank = lhs.CreateChild(nil, &yaml.Node{})
var newThing, err = mergeObjects(d, context, newBlank, lhs, multiplyPreferences{})
if err != nil {
return nil, err

View File

@@ -4,6 +4,26 @@ import (
"testing"
)
var doc1 = `list:
# Hi this is a comment.
# Hello this is another comment.
- "abc"`
var doc2 = `list2:
# This is yet another comment.
# Indeed this is yet another comment.
- "123"`
var docExpected = `D0, P[], (!!map)::list:
# Hi this is a comment.
# Hello this is another comment.
- "abc"
list2:
# This is yet another comment.
# Indeed this is yet another comment.
- "123"
`
var multiplyOperatorScenarios = []expressionScenario{
{
description: "Multiply integers",
@@ -12,6 +32,15 @@ var multiplyOperatorScenarios = []expressionScenario{
"D0, P[], (!!int)::12\n",
},
},
{
skipDoc: true,
document: doc1,
document2: doc2,
expression: `select(fi == 0) * select(fi == 1)`,
expected: []string{
docExpected,
},
},
{
skipDoc: true,
expression: `3 * 4.5`,
@@ -254,5 +283,5 @@ func TestMultiplyOperatorScenarios(t *testing.T) {
for _, tt := range multiplyOperatorScenarios {
testScenario(t, &tt)
}
documentScenarios(t, "Multiply", multiplyOperatorScenarios)
documentScenarios(t, "Multiply (Merge)", multiplyOperatorScenarios)
}

View File

@@ -3,11 +3,77 @@ package yqlib
import (
"container/list"
"fmt"
"regexp"
"strings"
"gopkg.in/yaml.v3"
)
func getSubstituteParameters(d *dataTreeNavigator, block *ExpressionNode, context Context) (string, string, error) {
regEx := ""
replacementText := ""
regExNodes, err := d.GetMatchingNodes(context, block.Lhs)
if err != nil {
return "", "", err
}
if regExNodes.MatchingNodes.Front() != nil {
regEx = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
}
log.Debug("regEx %v", regEx)
replacementNodes, err := d.GetMatchingNodes(context, block.Rhs)
if err != nil {
return "", "", err
}
if replacementNodes.MatchingNodes.Front() != nil {
replacementText = replacementNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
}
return regEx, replacementText, nil
}
func substitute(original string, regex *regexp.Regexp, replacement string) *yaml.Node {
replacedString := regex.ReplaceAllString(original, replacement)
return &yaml.Node{Kind: yaml.ScalarNode, Value: replacedString, Tag: "!!str"}
}
func substituteStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
//rhs block operator
//lhs of block = regex
//rhs of block = replacement expression
block := expressionNode.Rhs
regExStr, replacementText, err := getSubstituteParameters(d, block, context)
if err != nil {
return Context{}, err
}
regEx, err := regexp.Compile(regExStr)
if err != nil {
return Context{}, err
}
var results = list.New()
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
node := unwrapDoc(candidate.Node)
if node.Tag != "!!str" {
return Context{}, fmt.Errorf("cannot sustitute with %v, can only substitute strings", node.Tag)
}
targetNode := substitute(node.Value, regEx, replacementText)
result := candidate.CreateChild(nil, targetNode)
results.PushBack(result)
}
return context.ChildContext(results), nil
}
func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- joinStringOperator")
joinStr := ""
@@ -26,7 +92,7 @@ func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *E
candidate := el.Value.(*CandidateNode)
node := unwrapDoc(candidate.Node)
if node.Kind != yaml.SequenceNode {
return Context{}, fmt.Errorf("Cannot join with %v, can only join arrays of scalars", node.Tag)
return Context{}, fmt.Errorf("cannot join with %v, can only join arrays of scalars", node.Tag)
}
targetNode := join(node.Content, joinStr)
result := candidate.CreateChild(nil, targetNode)

View File

@@ -13,6 +13,24 @@ var stringsOperatorScenarios = []expressionScenario{
"D0, P[], (!!str)::cat; meow; 1; ; true\n",
},
},
{
description: "Substitute / Replace string",
subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)",
document: `a: dogs are great`,
expression: `.a |= sub("dogs", "cats")`,
expected: []string{
"D0, P[], (doc)::a: cats are great\n",
},
},
{
description: "Substitute / Replace string with regex",
subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)",
document: "a: cat\nb: heat",
expression: `.[] |= sub("([a])", "${1}r")`,
expected: []string{
"D0, P[], (doc)::a: cart\nb: heart\n",
},
},
{
description: "Split strings",
document: `"cat; meow; 1; ; true"`,

View File

@@ -0,0 +1,97 @@
package yqlib
import (
"fmt"
"strconv"
yaml "gopkg.in/yaml.v3"
)
func createSubtractOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {
return &ExpressionNode{Operation: &Operation{OperationType: subtractOpType},
Lhs: lhs,
Rhs: rhs}
}
func subtractAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
assignmentOp := &Operation{OperationType: assignOpType}
assignmentOp.UpdateAssign = true
selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createSubtractOp(selfExpression, expressionNode.Rhs)}
return d.GetMatchingNodes(context, assignmentOpNode)
}
func subtractOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("Subtract operator")
return crossFunction(d, context, expressionNode, subtract, false)
}
func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
lhs.Node = unwrapDoc(lhs.Node)
rhs.Node = unwrapDoc(rhs.Node)
lhsNode := lhs.Node
if lhsNode.Tag == "!!null" {
return lhs.CreateChild(nil, rhs.Node), nil
}
target := lhs.CreateChild(nil, &yaml.Node{})
switch lhsNode.Kind {
case yaml.MappingNode:
return nil, fmt.Errorf("Maps not yet supported for subtraction")
case yaml.SequenceNode:
return nil, fmt.Errorf("Sequences not yet supported for subtraction")
// target.Node.Kind = yaml.SequenceNode
// target.Node.Style = lhsNode.Style
// target.Node.Tag = "!!seq"
// target.Node.Content = append(lhsNode.Content, toNodes(rhs)...)
case yaml.ScalarNode:
if rhs.Node.Kind != yaml.ScalarNode {
return nil, fmt.Errorf("%v (%v) cannot be added to a %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag)
}
target.Node.Kind = yaml.ScalarNode
target.Node.Style = lhsNode.Style
return subtractScalars(target, lhsNode, rhs.Node)
}
return target, nil
}
func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*CandidateNode, error) {
if lhs.Tag == "!!str" {
return nil, fmt.Errorf("strings cannot be subtracted")
} else if lhs.Tag == "!!int" && rhs.Tag == "!!int" {
lhsNum, err := strconv.Atoi(lhs.Value)
if err != nil {
return nil, err
}
rhsNum, err := strconv.Atoi(rhs.Value)
if err != nil {
return nil, err
}
result := lhsNum - rhsNum
target.Node.Tag = "!!int"
target.Node.Value = fmt.Sprintf("%v", result)
} else if (lhs.Tag == "!!int" || lhs.Tag == "!!float") && (rhs.Tag == "!!int" || rhs.Tag == "!!float") {
lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
if err != nil {
return nil, err
}
rhsNum, err := strconv.ParseFloat(rhs.Value, 64)
if err != nil {
return nil, err
}
result := lhsNum - rhsNum
target.Node.Tag = "!!float"
target.Node.Value = fmt.Sprintf("%v", result)
} else {
return nil, fmt.Errorf("%v cannot be added to %v", lhs.Tag, rhs.Tag)
}
return target, nil
}

View File

@@ -0,0 +1,50 @@
package yqlib
import (
"testing"
)
var subtractOperatorScenarios = []expressionScenario{
{
description: "Number subtraction - float",
subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.",
document: `{a: 3, b: 4.5}`,
expression: `.a = .a - .b`,
expected: []string{
"D0, P[], (doc)::{a: -1.5, b: 4.5}\n",
},
},
{
description: "Number subtraction - float",
subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.",
document: `{a: 3, b: 4.5}`,
expression: `.a = .a - .b`,
expected: []string{
"D0, P[], (doc)::{a: -1.5, b: 4.5}\n",
},
},
{
description: "Number subtraction - int",
subdescription: "If both the lhs and rhs are ints then the expression will be calculated with ints.",
document: `{a: 3, b: 4}`,
expression: `.a = .a - .b`,
expected: []string{
"D0, P[], (doc)::{a: -1, b: 4}\n",
},
},
{
description: "Decrement numbers",
document: `{a: 3, b: 5}`,
expression: `.[] -= 1`,
expected: []string{
"D0, P[], (doc)::{a: 2, b: 4}\n",
},
},
}
func TestSubtractOperatorScenarios(t *testing.T) {
for _, tt := range subtractOperatorScenarios {
testScenario(t, &tt)
}
documentScenarios(t, "Subtract", subtractOperatorScenarios)
}

View File

@@ -24,7 +24,20 @@ func emptyOperator(d *dataTreeNavigator, context Context, expressionNode *Expres
type crossFunctionCalculation func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)
func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (Context, error) {
func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *CandidateNode, rhs Context, calculation crossFunctionCalculation, results *list.List) error {
for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() {
log.Debugf("Applying calc")
rhsCandidate := rightEl.Value.(*CandidateNode)
resultCandidate, err := calculation(d, context, lhsCandidate, rhsCandidate)
if err != nil {
return err
}
results.PushBack(resultCandidate)
}
return nil
}
func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation, calcWhenEmpty bool) (Context, error) {
var results = list.New()
lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
if err != nil {
@@ -38,24 +51,33 @@ func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *Expressi
return Context{}, err
}
for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
lhsCandidate := el.Value.(*CandidateNode)
for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() {
log.Debugf("Applying calc")
rhsCandidate := rightEl.Value.(*CandidateNode)
resultCandidate, err := calculation(d, context, lhsCandidate, rhsCandidate)
if calcWhenEmpty && lhs.MatchingNodes.Len() == 0 {
if rhs.MatchingNodes.Len() == 0 {
resultCandidate, err := calculation(d, context, nil, nil)
if err != nil {
return Context{}, err
}
results.PushBack(resultCandidate)
}
err := resultsForRhs(d, context, nil, rhs, calculation, results)
if err != nil {
return Context{}, err
}
}
for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
lhsCandidate := el.Value.(*CandidateNode)
err := resultsForRhs(d, context, lhsCandidate, rhs, calculation, results)
if err != nil {
return Context{}, err
}
}
return context.ChildContext(results), nil
}
func crossFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (Context, error) {
func crossFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation, calcWhenEmpty bool) (Context, error) {
var results = list.New()
var evaluateAllTogether = true
@@ -66,11 +88,11 @@ func crossFunction(d *dataTreeNavigator, context Context, expressionNode *Expres
}
}
if evaluateAllTogether {
return doCrossFunc(d, context, expressionNode, calculation)
return doCrossFunc(d, context, expressionNode, calculation, calcWhenEmpty)
}
for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
innerResults, err := doCrossFunc(d, context.SingleChildContext(matchEl.Value.(*CandidateNode)), expressionNode, calculation)
innerResults, err := doCrossFunc(d, context.SingleChildContext(matchEl.Value.(*CandidateNode)), expressionNode, calculation, calcWhenEmpty)
if err != nil {
return Context{}, err
}

View File

@@ -34,7 +34,7 @@ func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEn
unwrapScalar: unwrapScalar,
colorsEnabled: colorsEnabled,
indent: indent,
printDocSeparators: printDocSeparators,
printDocSeparators: !outputToJSON && printDocSeparators,
firstTimePrinting: true,
treeNavigator: NewDataTreeNavigator(),
}

View File

@@ -130,7 +130,9 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
func TestPrinterMultipleDocsJson(t *testing.T) {
var output bytes.Buffer
var writer = bufio.NewWriter(&output)
printer := NewPrinter(writer, true, true, false, 0, false)
// note printDocSeparators is true, it should still not print document separators
// when outputing JSON.
printer := NewPrinter(writer, true, true, false, 0, true)
inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
if err != nil {

View File

@@ -2,6 +2,7 @@
- increment version in snapcraft.yaml
- increment version in github-action/Dockerfile
- make sure local build passes
- run gosec (manual because docker platforms)
- commit version update changes
- tag git with same version number
- commit vX tag - this will trigger github actions

View File

@@ -10,12 +10,3 @@ else
./bin/golangci-lint run --timeout=5m
fi
# ./bin/golangci-lint \
# --tests \
# --vendor \
# --disable=aligncheck \
# --disable=gotype \
# --disable=goconst \
# --disable=gocyclo \
# --deadline=300s \
# ./...

View File

@@ -1,4 +1,5 @@
#!/bin/sh
set -ex
go get golang.org/x/tools/cmd/goimports
wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.24.0
wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.37.1
# wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.6.1

11
scripts/secure.sh Executable file
View File

@@ -0,0 +1,11 @@
#!/bin/bash
set -o errexit
set -o pipefail
if command -v gosec &> /dev/null
then
gosec ${PWD}
else
./bin/gosec ${PWD}
fi

View File

@@ -1,5 +1,5 @@
name: yq
version: '4.6.0'
version: '4.7.0'
summary: A lightweight and portable command-line YAML processor
description: |
The aim of the project is to be the jq or sed of yaml files.