1
0
mirror of https://github.com/taigrr/yq synced 2025-01-18 04:53:17 -08:00

Compare commits

...

20 Commits

Author SHA1 Message Date
Mike Farah
9e9e15df73 More scenarios 2020-12-27 23:00:46 +11:00
Mike Farah
6cc6fdf322 Cleaning code 2020-12-27 22:56:15 +11:00
Mike Farah
a88c2dc5d3 Traverse Array Operator 2020-12-27 22:48:20 +11:00
Mike Farah
ea231006ed Refactoring traverse 2020-12-27 09:55:08 +11:00
Mike Farah
80f187f1a4 Refactoring traverse 2020-12-27 09:51:34 +11:00
Mike Farah
98e8b3479f Fixed nested array splat path 2020-12-25 12:49:05 +11:00
Mike Farah
eb539ff326 Fixed doc links 2020-12-23 10:37:51 +11:00
Mike Farah
c09f7aa707 Cleaning up docs 2020-12-23 10:30:13 +11:00
Mike Farah
7f5c380d16 v4.1.0 2020-12-23 10:23:54 +11:00
Mike Farah
6a05e517f1 Updated release instructions, remove gate for release 2020-12-23 10:23:09 +11:00
Mike Farah
8ee6f7dc1a fixing xcompile for git action 2020-12-22 22:50:01 +11:00
Mike Farah
8bd54cd603 fixing xcompile for git action 2020-12-22 22:31:28 +11:00
Mike Farah
f2f7b6db0f only tar executable files 2020-12-22 20:50:52 +11:00
Mike Farah
e082fee5d4 Fixed rhash call 2020-12-22 20:37:35 +11:00
Mike Farah
412911561f Fixed xcompile.sh 2020-12-22 20:23:13 +11:00
Mike Farah
52ae633cb7 trialing github release actions 2020-12-22 20:05:23 +11:00
Mike Farah
9356ca59bb trialing github release actions 2020-12-22 20:03:01 +11:00
Mike Farah
6dec167f74 trialing github release actions 2020-12-22 20:01:21 +11:00
Mike Farah
00f6981314 trialing github release actions 2020-12-22 19:56:56 +11:00
Mike Farah
4c60a2a967 trialing github release actions 2020-12-22 19:52:44 +11:00
23 changed files with 481 additions and 286 deletions

View File

@@ -1,4 +1,4 @@
name: Publish image to Dockerhub
name: Release YQ
on:
push:
tags:
@@ -6,14 +6,17 @@ on:
jobs:
publishGitRelease:
environment: gitrelease
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
# steps for building assets
- uses: actions/setup-go@v2
with:
go-version: '^1.15'
- name: Cross compile
run: ./scripts/xcompile.sh
run: |
sudo apt-get install rhash -y
go get github.com/mitchellh/gox
./scripts/xcompile.sh
- name: Create Release
id: create_release

View File

@@ -80,7 +80,7 @@ yq() {
### Go Get:
```
GO111MODULE=on go get github.com/mikefarah/yq/v4
GO111MODULE=on go get github.com/mikefarah/yq
```
## Community Supported Installation methods
@@ -126,7 +126,7 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
- Colorized yaml output
- [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/v/v4.x/traverse)
- [Sort yaml by keys](https://mikefarah.gitbook.io/yq/v/v4.x/sort-keys)
- Manipulate yaml [comments](https://app.gitbook.com/@mikefarah/s/yq/v/v4.x/comment-operators), [styling](https://app.gitbook.com/@mikefarah/s/yq/v/v4.x/style), [tags](https://app.gitbook.com/@mikefarah/s/yq/v/v4.x/tag) and [anchors](https://app.gitbook.com/@mikefarah/s/yq/v/v4.x/explode).
- Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/comment-operators), [styling](https://mikefarah.gitbook.io/yq/style), [tags](https://mikefarah.gitbook.io/yq/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/anchor-and-alias-operators).
- [Update yaml inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
- [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/v/v4.x/select#select-and-update-matching-values-in-map)
- Keeps yaml formatting and comments when updating (though there are issues with whitespace)

View File

@@ -11,7 +11,7 @@ var (
GitDescribe string
// Version is main version number that is being run at the moment.
Version = "4.0.0"
Version = "4.1.0"
// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release

View File

@@ -21,6 +21,14 @@ func (n *CandidateNode) GetKey() string {
return fmt.Sprintf("%v - %v", n.Document, n.Path)
}
func (n *CandidateNode) CreateChildPath(path interface{}) []interface{} {
//don't use append as they may actually modify the path of the orignal node!
newPath := make([]interface{}, len(n.Path)+1)
copy(newPath, n.Path)
newPath[len(n.Path)] = path
return newPath
}
func (n *CandidateNode) Copy() (*CandidateNode, error) {
clone := &CandidateNode{}
err := copier.Copy(clone, n)

View File

@@ -112,7 +112,7 @@ a:
```
then
```bash
yq eval '(.a.[] | select(. == "apple")) = "frog"' sample.yml
yq eval '(.a[] | select(. == "apple")) = "frog"' sample.yml
```
will output
```yaml

View File

@@ -106,7 +106,7 @@ b: *cat
```
then
```bash
yq eval '.b.[]' sample.yml
yq eval '.b[]' sample.yml
```
will output
```yaml
@@ -290,7 +290,7 @@ foobar:
```
then
```bash
yq eval '.foobar.[]' sample.yml
yq eval '.foobar[]' sample.yml
```
will output
```yaml
@@ -356,7 +356,7 @@ foobar:
```
then
```bash
yq eval '.foobarList.[]' sample.yml
yq eval '.foobarList[]' sample.yml
```
will output
```yaml
@@ -366,3 +366,21 @@ bar_thing
foobarList_c
```
## Select multiple indices
Given a sample.yml file of:
```yaml
a:
- a
- b
- c
```
then
```bash
yq eval '.a[0, 2]' sample.yml
```
will output
```yaml
a
c
```

View File

@@ -65,6 +65,7 @@ var SortKeys = &OperationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Han
var CollectObject = &OperationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: CollectObjectOperator}
var TraversePath = &OperationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
var TraverseArray = &OperationType{Type: "TRAVERSE_ARRAY", NumArgs: 1, Precedence: 50, Handler: TraverseArrayOperator}
var DocumentFilter = &OperationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: SelfOperator}

View File

@@ -118,5 +118,5 @@ func TestAnchorAliaseOperatorScenarios(t *testing.T) {
for _, tt := range anchorOperatorScenarios {
testScenario(t, &tt)
}
documentScenarios(t, "Anchor and Aliases Operators", anchorOperatorScenarios)
documentScenarios(t, "Anchor and Alias Operators", anchorOperatorScenarios)
}

View File

@@ -80,6 +80,14 @@ var assignOperatorScenarios = []expressionScenario{
{
description: "Update selected results",
document: `{a: {b: apple, c: cactus}}`,
expression: `(.a[] | select(. == "apple")) = "frog"`,
expected: []string{
"D0, P[], (doc)::{a: {b: frog, c: cactus}}\n",
},
},
{
skipDoc: true,
document: `{a: {b: apple, c: cactus}}`,
expression: `(.a.[] | select(. == "apple")) = "frog"`,
expected: []string{
"D0, P[], (doc)::{a: {b: frog, c: cactus}}\n",

View File

@@ -55,7 +55,7 @@ func CollectObjectOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *
func createChildCandidate(candidate *CandidateNode, index int) *CandidateNode {
return &CandidateNode{
Document: candidate.Document,
Path: append(candidate.Path, index),
Path: candidate.CreateChildPath(index),
Filename: candidate.Filename,
Node: candidate.Node.Content[index],
}

View File

@@ -75,7 +75,7 @@ func deleteFromMap(candidate *CandidateNode, childPath interface{}) {
childCandidate := &CandidateNode{
Node: value,
Document: candidate.Document,
Path: append(candidate.Path, key.Value),
Path: candidate.CreateChildPath(key.Value),
}
shouldDelete := key.Value == childPath

View File

@@ -13,6 +13,23 @@ var pathOperatorScenarios = []expressionScenario{
"D0, P[a b], (!!seq)::- a\n- b\n",
},
},
{
skipDoc: true,
document: `a:
b:
c:
- 0
- 1
- 2
- 3`,
expression: `.a.b.c.[]`,
expected: []string{
"D0, P[a b c 0], (!!int)::0\n",
"D0, P[a b c 1], (!!int)::1\n",
"D0, P[a b c 2], (!!int)::2\n",
"D0, P[a b c 3], (!!int)::3\n",
},
},
{
description: "Get map key",
document: `{a: {b: cat}}`,

View File

@@ -1,13 +1,12 @@
package yqlib
import (
"fmt"
"container/list"
"fmt"
"strconv"
"github.com/elliotchance/orderedmap"
"gopkg.in/yaml.v3"
yaml "gopkg.in/yaml.v3"
)
type TraversePreferences struct {
@@ -15,32 +14,25 @@ type TraversePreferences struct {
}
func Splat(d *dataTreeNavigator, matches *list.List) (*list.List, error) {
preferences := &TraversePreferences{DontFollowAlias: true}
splatOperation := &Operation{OperationType: TraversePath, Value: "[]", Preferences: preferences}
splatTreeNode := &PathTreeNode{Operation: splatOperation}
return TraversePathOperator(d, matches, splatTreeNode)
return traverseNodesWithArrayIndices(matches, make([]*yaml.Node, 0), false)
}
func TraversePathOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
log.Debugf("-- Traversing")
var matchingNodeMap = list.New()
var newNodes []*CandidateNode
var err error
for el := matchMap.Front(); el != nil; el = el.Next() {
newNodes, err = traverse(d, el.Value.(*CandidateNode), pathNode.Operation)
newNodes, err := traverse(d, el.Value.(*CandidateNode), pathNode.Operation)
if err != nil {
return nil, err
}
for _, n := range newNodes {
matchingNodeMap.PushBack(n)
}
matchingNodeMap.PushBackList(newNodes)
}
return matchingNodeMap, nil
}
func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Operation) ([]*CandidateNode, error) {
func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Operation) (*list.List, error) {
log.Debug("Traversing %v", NodeToString(matchingNode))
value := matchingNode.Node
@@ -61,34 +53,12 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Oper
switch value.Kind {
case yaml.MappingNode:
log.Debug("its a map with %v entries", len(value.Content)/2)
var newMatches = orderedmap.NewOrderedMap()
err := traverseMap(newMatches, matchingNode, operation)
followAlias := true
if err != nil {
return nil, err
if operation.Preferences != nil {
followAlias = !operation.Preferences.(*TraversePreferences).DontFollowAlias
}
if newMatches.Len() == 0 {
//no matches, create one automagically
valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}
node := matchingNode.Node
node.Content = append(node.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: operation.StringValue}, valueNode)
candidateNode := &CandidateNode{
Node: valueNode,
Path: append(matchingNode.Path, operation.StringValue),
Document: matchingNode.Document,
}
newMatches.Set(candidateNode.GetKey(), candidateNode)
}
arrayMatches := make([]*CandidateNode, newMatches.Len())
i := 0
for el := newMatches.Front(); el != nil; el = el.Next() {
arrayMatches[i] = el.Value.(*CandidateNode)
i++
}
return arrayMatches, nil
return traverseMap(matchingNode, operation.StringValue, followAlias, false)
case yaml.SequenceNode:
log.Debug("its a sequence of %v things!", len(value.Content))
@@ -102,104 +72,117 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Oper
log.Debug("digging into doc node")
return traverse(d, &CandidateNode{
Node: matchingNode.Node.Content[0],
Filename: matchingNode.Filename,
FileIndex: matchingNode.FileIndex,
Document: matchingNode.Document}, operation)
default:
return nil, nil
return list.New(), nil
}
}
func keyMatches(key *yaml.Node, pathNode *Operation) bool {
return pathNode.Value == "[]" || Match(key.Value, pathNode.StringValue)
}
func TraverseArrayOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
// rhs is a collect expression that will yield indexes to retreive of the arrays
func traverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, operation *Operation) error {
// value.Content is a concatenated array of key, value,
// so keys are in the even indexes, values in odd.
// merge aliases are defined first, but we only want to traverse them
// if we don't find a match directly on this node first.
//TODO ALIASES, auto creation?
node := candidate.Node
followAlias := true
if operation.Preferences != nil {
followAlias = !operation.Preferences.(*TraversePreferences).DontFollowAlias
}
var contents = node.Content
for index := 0; index < len(contents); index = index + 2 {
key := contents[index]
value := contents[index+1]
log.Debug("checking %v (%v)", key.Value, key.Tag)
//skip the 'merge' tag, find a direct match first
if key.Tag == "!!merge" && followAlias {
log.Debug("Merge anchor")
err := traverseMergeAnchor(newMatches, candidate, value, operation)
rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
if err != nil {
return err
}
} else if keyMatches(key, operation) {
log.Debug("MATCHED")
candidateNode := &CandidateNode{
Node: value,
Path: append(candidate.Path, key.Value),
Document: candidate.Document,
}
newMatches.Set(candidateNode.GetKey(), candidateNode)
}
return nil, err
}
return nil
var indicesToTraverse = rhs.Front().Value.(*CandidateNode).Node.Content
return traverseNodesWithArrayIndices(matchingNodes, indicesToTraverse, true)
}
func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, operation *Operation) error {
switch value.Kind {
case yaml.AliasNode:
candidateNode := &CandidateNode{
Node: value.Alias,
Path: originalCandidate.Path,
Document: originalCandidate.Document,
}
return traverseMap(newMatches, candidateNode, operation)
case yaml.SequenceNode:
for _, childValue := range value.Content {
err := traverseMergeAnchor(newMatches, originalCandidate, childValue, operation)
func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, followAlias bool) (*list.List, error) {
var matchingNodeMap = list.New()
for el := matchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
newNodes, err := traverseArrayIndices(candidate, indicesToTraverse, followAlias)
if err != nil {
return err
return nil, err
}
matchingNodeMap.PushBackList(newNodes)
}
}
return nil
return matchingNodeMap, nil
}
func traverseArray(candidate *CandidateNode, operation *Operation) ([]*CandidateNode, error) {
log.Debug("operation Value %v", operation.Value)
if operation.Value == "[]" {
func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, followAlias bool) (*list.List, error) { // call this if doc / alias like the other traverse
node := matchingNode.Node
if node.Tag == "!!null" {
log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
// auto vivification, make it into an empty array
node.Tag = ""
node.Kind = yaml.SequenceNode
}
var contents = candidate.Node.Content
var newMatches = make([]*CandidateNode, len(contents))
if node.Kind == yaml.AliasNode {
matchingNode.Node = node.Alias
return traverseArrayIndices(matchingNode, indicesToTraverse, followAlias)
} else if node.Kind == yaml.SequenceNode {
return traverseArrayWithIndices(matchingNode, indicesToTraverse)
} else if node.Kind == yaml.MappingNode {
return traverseMapWithIndices(matchingNode, indicesToTraverse, followAlias)
} else if node.Kind == yaml.DocumentNode {
return traverseArrayIndices(&CandidateNode{
Node: matchingNode.Node.Content[0],
Filename: matchingNode.Filename,
FileIndex: matchingNode.FileIndex,
Document: matchingNode.Document}, indicesToTraverse, followAlias)
}
log.Debugf("OperatorArrayTraverse skipping %v as its a %v", matchingNode, node.Tag)
return list.New(), nil
}
func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, followAlias bool) (*list.List, error) {
if len(indices) == 0 {
return traverseMap(candidate, "", followAlias, true)
}
var matchingNodeMap = list.New()
for _, indexNode := range indices {
log.Debug("traverseMapWithIndices: %v", indexNode.Value)
newNodes, err := traverseMap(candidate, indexNode.Value, followAlias, false)
if err != nil {
return nil, err
}
matchingNodeMap.PushBackList(newNodes)
}
return matchingNodeMap, nil
}
func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*list.List, error) {
log.Debug("traverseArrayWithIndices")
var newMatches = list.New()
node := UnwrapDoc(candidate.Node)
if len(indices) == 0 {
log.Debug("splatting")
var index int64
for index = 0; index < int64(len(contents)); index = index + 1 {
newMatches[index] = &CandidateNode{
for index = 0; index < int64(len(node.Content)); index = index + 1 {
newMatches.PushBack(&CandidateNode{
Document: candidate.Document,
Path: append(candidate.Path, index),
Node: contents[index],
}
Path: candidate.CreateChildPath(index),
Node: node.Content[index],
})
}
return newMatches, nil
}
switch operation.Value.(type) {
case int64:
index := operation.Value.(int64)
for _, indexNode := range indices {
log.Debug("traverseArrayWithIndices: '%v'", indexNode.Value)
index, err := strconv.ParseInt(indexNode.Value, 10, 64)
if err != nil {
return nil, fmt.Errorf("Cannot index array with '%v' (%v)", indexNode.Value, err)
}
indexToUse := index
contentLength := int64(len(candidate.Node.Content))
contentLength := int64(len(node.Content))
for contentLength <= index {
candidate.Node.Content = append(candidate.Node.Content, &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"})
contentLength = int64(len(candidate.Node.Content))
node.Content = append(node.Content, &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"})
contentLength = int64(len(node.Content))
}
if indexToUse < 0 {
@@ -210,14 +193,107 @@ func traverseArray(candidate *CandidateNode, operation *Operation) ([]*Candidate
return nil, fmt.Errorf("Index [%v] out of range, array size is %v", index, contentLength)
}
return []*CandidateNode{&CandidateNode{
Node: candidate.Node.Content[indexToUse],
newMatches.PushBack(&CandidateNode{
Node: node.Content[indexToUse],
Document: candidate.Document,
Path: append(candidate.Path, index),
}}, nil
default:
log.Debug("argument not an int (%v), no array matches", operation.Value)
return nil, nil
Path: candidate.CreateChildPath(index),
})
}
return newMatches, nil
}
func keyMatches(key *yaml.Node, wantedKey string) bool {
return Match(key.Value, wantedKey)
}
func traverseMap(matchingNode *CandidateNode, key string, followAlias bool, splat bool) (*list.List, error) {
var newMatches = orderedmap.NewOrderedMap()
err := doTraverseMap(newMatches, matchingNode, key, followAlias, splat)
if err != nil {
return nil, err
}
if newMatches.Len() == 0 {
//no matches, create one automagically
valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}
node := matchingNode.Node
node.Content = append(node.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: key}, valueNode)
candidateNode := &CandidateNode{
Node: valueNode,
Path: append(matchingNode.Path, key),
Document: matchingNode.Document,
}
newMatches.Set(candidateNode.GetKey(), candidateNode)
}
results := list.New()
i := 0
for el := newMatches.Front(); el != nil; el = el.Next() {
results.PushBack(el.Value)
i++
}
return results, nil
}
func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, followAlias bool, splat bool) error {
// value.Content is a concatenated array of key, value,
// so keys are in the even indexes, values in odd.
// merge aliases are defined first, but we only want to traverse them
// if we don't find a match directly on this node first.
node := candidate.Node
var contents = node.Content
for index := 0; index < len(contents); index = index + 2 {
key := contents[index]
value := contents[index+1]
log.Debug("checking %v (%v)", key.Value, key.Tag)
//skip the 'merge' tag, find a direct match first
if key.Tag == "!!merge" && followAlias {
log.Debug("Merge anchor")
err := traverseMergeAnchor(newMatches, candidate, value, wantedKey, splat)
if err != nil {
return err
}
} else if splat || keyMatches(key, wantedKey) {
log.Debug("MATCHED")
candidateNode := &CandidateNode{
Node: value,
Path: candidate.CreateChildPath(key.Value),
Document: candidate.Document,
}
newMatches.Set(candidateNode.GetKey(), candidateNode)
}
}
return nil
}
func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, splat bool) error {
switch value.Kind {
case yaml.AliasNode:
candidateNode := &CandidateNode{
Node: value.Alias,
Path: originalCandidate.Path,
Document: originalCandidate.Document,
}
return doTraverseMap(newMatches, candidateNode, wantedKey, true, splat)
case yaml.SequenceNode:
for _, childValue := range value.Content {
err := traverseMergeAnchor(newMatches, originalCandidate, childValue, wantedKey, splat)
if err != nil {
return err
}
}
}
return nil
}
func traverseArray(candidate *CandidateNode, operation *Operation) (*list.List, error) {
log.Debug("operation Value %v", operation.Value)
indices := []*yaml.Node{&yaml.Node{Value: operation.StringValue}}
return traverseArrayWithIndices(candidate, indices)
}

View File

@@ -65,7 +65,7 @@ var traversePathOperatorScenarios = []expressionScenario{
},
{
skipDoc: true,
document: `{}`,
document: ``,
expression: `.[1].a`,
expected: []string{
"D0, P[1 a], (!!null)::null\n",
@@ -137,6 +137,14 @@ var traversePathOperatorScenarios = []expressionScenario{
{
description: "Traversing aliases with splat",
document: `{a: &cat {c: frog}, b: *cat}`,
expression: `.b[]`,
expected: []string{
"D0, P[b c], (!!str)::frog\n",
},
},
{
skipDoc: true,
document: `{a: &cat {c: frog}, b: *cat}`,
expression: `.b.[]`,
expected: []string{
"D0, P[b c], (!!str)::frog\n",
@@ -150,12 +158,6 @@ var traversePathOperatorScenarios = []expressionScenario{
"D0, P[b c], (!!str)::frog\n",
},
},
{
skipDoc: true,
document: `[1,2,3]`,
expression: `.b`,
expected: []string{},
},
{
description: "Traversing arrays by index",
document: `[1,2,3]`,
@@ -215,6 +217,16 @@ var traversePathOperatorScenarios = []expressionScenario{
{
description: "Splatting merge anchors",
document: mergeDocSample,
expression: `.foobar[]`,
expected: []string{
"D0, P[foobar c], (!!str)::foo_c\n",
"D0, P[foobar a], (!!str)::foo_a\n",
"D0, P[foobar thing], (!!str)::foobar_thing\n",
},
},
{
skipDoc: true,
document: mergeDocSample,
expression: `.foobar.[]`,
expected: []string{
"D0, P[foobar c], (!!str)::foo_c\n",
@@ -266,6 +278,17 @@ var traversePathOperatorScenarios = []expressionScenario{
{
description: "Splatting merge anchor lists",
document: mergeDocSample,
expression: `.foobarList[]`,
expected: []string{
"D0, P[foobarList b], (!!str)::bar_b\n",
"D0, P[foobarList a], (!!str)::foo_a\n",
"D0, P[foobarList thing], (!!str)::bar_thing\n",
"D0, P[foobarList c], (!!str)::foobarList_c\n",
},
},
{
skipDoc: true,
document: mergeDocSample,
expression: `.foobarList.[]`,
expected: []string{
"D0, P[foobarList b], (!!str)::bar_b\n",
@@ -274,6 +297,112 @@ var traversePathOperatorScenarios = []expressionScenario{
"D0, P[foobarList c], (!!str)::foobarList_c\n",
},
},
{
skipDoc: true,
document: `[a,b,c]`,
expression: `.[]`,
expected: []string{
"D0, P[0], (!!str)::a\n",
"D0, P[1], (!!str)::b\n",
"D0, P[2], (!!str)::c\n",
},
},
{
skipDoc: true,
document: `[a,b,c]`,
expression: `[]`,
expected: []string{
"D0, P[], (!!seq)::[]\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a[0]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
},
},
{
description: "Select multiple indices",
document: `{a: [a,b,c]}`,
expression: `.a[0, 2]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
"D0, P[a 2], (!!str)::c\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a.[0, 2]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
"D0, P[a 2], (!!str)::c\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a[-1]`,
expected: []string{
"D0, P[a -1], (!!str)::c\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a.[-1]`,
expected: []string{
"D0, P[a -1], (!!str)::c\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a[-2]`,
expected: []string{
"D0, P[a -2], (!!str)::b\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a.[-2]`,
expected: []string{
"D0, P[a -2], (!!str)::b\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a[]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
"D0, P[a 1], (!!str)::b\n",
"D0, P[a 2], (!!str)::c\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a.[]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
"D0, P[a 1], (!!str)::b\n",
"D0, P[a 2], (!!str)::c\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a | .[]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
"D0, P[a 1], (!!str)::b\n",
"D0, P[a 2], (!!str)::c\n",
},
},
}
func TestTraversePathOperatorScenarios(t *testing.T) {

View File

@@ -38,7 +38,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
if s.document != "" {
inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml", 0)
if err != nil {
t.Error(err, s.document)
t.Error(err, s.document, s.expression)
return
}
} else {
@@ -55,7 +55,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
results, err = treeNavigator.GetMatchingNodes(inputs, node)
if err != nil {
t.Error(err)
t.Error(fmt.Errorf("%v: %v", err, s.expression))
return
}
test.AssertResultComplexWithContext(t, s.expected, resultsToString(results), fmt.Sprintf("exp: %v\ndoc: %v", s.expression, s.document))
@@ -167,17 +167,17 @@ func documentScenarios(t *testing.T, title string, scenarios []expressionScenari
if s.document != "" {
node, err := treeCreator.ParsePath(s.expression)
if err != nil {
t.Error(err)
t.Error(err, s.expression)
}
err = streamEvaluator.Evaluate("sample.yaml", strings.NewReader(formattedDoc), node, printer)
if err != nil {
t.Error(err)
t.Error(err, s.expression)
}
} else {
err = streamEvaluator.EvaluateNew(s.expression, printer)
if err != nil {
t.Error(err)
t.Error(err, s.expression)
}
}

View File

@@ -17,20 +17,35 @@ var pathTests = []struct {
append(make([]interface{}, 0), "[", "]"),
append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE"),
},
{
`.[]`,
append(make([]interface{}, 0), "TRAVERSE_ARRAY", "[", "]"),
append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
},
{
`.a[]`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "[]"),
append(make([]interface{}, 0), "a", "[]", "SHORT_PIPE"),
append(make([]interface{}, 0), "a", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "]"),
append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
},
{
`.a.[]`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "[]"),
append(make([]interface{}, 0), "a", "[]", "SHORT_PIPE"),
append(make([]interface{}, 0), "a", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "]"),
append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
},
{
`.a[0]`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
},
{
`.a.[0]`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
},
{
`.a[].c`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "[]", "SHORT_PIPE", "c"),
append(make([]interface{}, 0), "a", "[]", "SHORT_PIPE", "c", "SHORT_PIPE"),
append(make([]interface{}, 0), "a", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "]", "SHORT_PIPE", "c"),
append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "c", "SHORT_PIPE"),
},
{
`[3]`,
@@ -44,18 +59,18 @@ var pathTests = []struct {
},
{
`.a | .[].b == "apple"`,
append(make([]interface{}, 0), "a", "PIPE", "[]", "SHORT_PIPE", "b", "EQUALS", "apple (string)"),
append(make([]interface{}, 0), "a", "[]", "b", "SHORT_PIPE", "apple (string)", "EQUALS", "PIPE"),
append(make([]interface{}, 0), "a", "PIPE", "TRAVERSE_ARRAY", "[", "]", "SHORT_PIPE", "b", "EQUALS", "apple (string)"),
append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "apple (string)", "EQUALS", "PIPE"),
},
{
`(.a | .[].b) == "apple"`,
append(make([]interface{}, 0), "(", "a", "PIPE", "[]", "SHORT_PIPE", "b", ")", "EQUALS", "apple (string)"),
append(make([]interface{}, 0), "a", "[]", "b", "SHORT_PIPE", "PIPE", "apple (string)", "EQUALS"),
append(make([]interface{}, 0), "(", "a", "PIPE", "TRAVERSE_ARRAY", "[", "]", "SHORT_PIPE", "b", ")", "EQUALS", "apple (string)"),
append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "PIPE", "apple (string)", "EQUALS"),
},
{
`.[] | select(. == "*at")`,
append(make([]interface{}, 0), "[]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at (string)", ")"),
append(make([]interface{}, 0), "[]", "SELF", "*at (string)", "EQUALS", "SELECT", "PIPE"),
append(make([]interface{}, 0), "TRAVERSE_ARRAY", "[", "]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at (string)", ")"),
append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SELF", "*at (string)", "EQUALS", "SELECT", "PIPE"),
},
{
`[true]`,
@@ -89,8 +104,8 @@ var pathTests = []struct {
},
{
`{.a: .c, .b.[]: .f.g.[]}`,
append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "SHORT_PIPE", "[]", "CREATE_MAP", "f", "SHORT_PIPE", "g", "SHORT_PIPE", "[]", "}"),
append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "[]", "SHORT_PIPE", "f", "g", "SHORT_PIPE", "[]", "SHORT_PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "SHORT_PIPE"),
append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "]", "CREATE_MAP", "f", "SHORT_PIPE", "g", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "]", "}"),
append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "f", "g", "SHORT_PIPE", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "SHORT_PIPE"),
},
{
`explode(.a.b)`,
@@ -122,7 +137,6 @@ var pathTests = []struct {
append(make([]interface{}, 0), "SELF", "ASSIGN_COMMENT", "str (string)"),
append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_COMMENT"),
},
{
`.a.b tag="!!str"`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "b", "ASSIGN_TAG", "!!str (string)"),

View File

@@ -22,7 +22,7 @@ const (
CloseCollect
OpenCollectObject
CloseCollectObject
SplatOrEmptyCollect
TraverseArrayCollect
)
type Token struct {
@@ -49,8 +49,9 @@ func (t *Token) toString() string {
return "{"
} else if t.TokenType == CloseCollectObject {
return "}"
} else if t.TokenType == SplatOrEmptyCollect {
return "[]?"
} else if t.TokenType == TraverseArrayCollect {
return ".["
} else {
return "NFI"
}
@@ -114,23 +115,6 @@ func unwrap(value string) string {
return value[1 : len(value)-1]
}
func arrayIndextoken(precedingDot bool) lex.Action {
return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
var numberString = string(m.Bytes)
startIndex := 1
if precedingDot {
startIndex = 2
}
numberString = numberString[startIndex : len(numberString)-1]
var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) // nolint
if errParsingInt != nil {
return nil, errParsingInt
}
op := &Operation{OperationType: TraversePath, Value: number, StringValue: numberString}
return &Token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil
}
}
func numberValue() lex.Action {
return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
var numberString = string(m.Bytes)
@@ -188,7 +172,7 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`\(`), literalToken(OpenBracket, false))
lexer.Add([]byte(`\)`), literalToken(CloseBracket, true))
lexer.Add([]byte(`\.\[\]`), pathToken(false))
lexer.Add([]byte(`\.\[`), literalToken(TraverseArrayCollect, false))
lexer.Add([]byte(`\.\.`), opToken(RecursiveDescent))
lexer.Add([]byte(`,`), opToken(Union))
@@ -231,8 +215,6 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`\s*\|=\s*`), opTokenWithPrefs(Assign, nil, &AssignOpPreferences{true}))
lexer.Add([]byte(`\.\[-?[0-9]+\]`), arrayIndextoken(true))
lexer.Add([]byte("( |\t|\n|\r)+"), skip)
lexer.Add([]byte(`d[0-9]+`), documentToken())
@@ -254,8 +236,6 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`"[^"]*"`), stringValue(true))
lexer.Add([]byte(`\[\]`), literalToken(SplatOrEmptyCollect, true))
lexer.Add([]byte(`\[`), literalToken(OpenCollect, false))
lexer.Add([]byte(`\]`), literalToken(CloseCollect, true))
lexer.Add([]byte(`\{`), literalToken(OpenCollectObject, false))
@@ -324,24 +304,16 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) {
func (p *pathTokeniser) handleToken(tokens []*Token, index int, postProcessedTokens []*Token) (tokensAccum []*Token, skipNextToken bool) {
skipNextToken = false
token := tokens[index]
if token.TokenType == SplatOrEmptyCollect {
if index > 0 && tokens[index-1].TokenType == OperationToken &&
tokens[index-1].Operation.OperationType == TraversePath {
// must be a splat without a preceding dot , e.g. .a[]
// lets put a pipe in front of it, and convert it to a traverse "[]" token
pipeOp := &Operation{OperationType: ShortPipe, Value: "PIPE"}
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: pipeOp})
if token.TokenType == TraverseArrayCollect {
//need to put a traverse array then a collect token
// do this by adding traverse then converting token to collect
traverseOp := &Operation{OperationType: TraversePath, Value: "[]", StringValue: "[]"}
token = &Token{TokenType: OperationToken, Operation: traverseOp, CheckForPostTraverse: true}
op := &Operation{OperationType: TraverseArray, StringValue: "TRAVERSE_ARRAY"}
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
token = &Token{TokenType: OpenCollect}
} else {
// gotta be a collect empty array, we need to split this into two tokens
// one OpenCollect, the other CloseCollect
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OpenCollect})
token = &Token{TokenType: CloseCollect, CheckForPostTraverse: true}
}
}
if index != len(tokens)-1 && token.AssignOperation != nil &&
@@ -359,5 +331,21 @@ func (p *pathTokeniser) handleToken(tokens []*Token, index int, postProcessedTok
op := &Operation{OperationType: ShortPipe, Value: "PIPE"}
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
}
if index != len(tokens)-1 && token.CheckForPostTraverse &&
tokens[index+1].TokenType == OpenCollect {
op := &Operation{OperationType: ShortPipe, Value: "PIPE"}
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
op = &Operation{OperationType: TraverseArray}
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
}
if index != len(tokens)-1 && token.CheckForPostTraverse &&
tokens[index+1].TokenType == TraverseArrayCollect {
op := &Operation{OperationType: ShortPipe, Value: "PIPE"}
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
}
return postProcessedTokens, skipNextToken
}

View File

@@ -1,16 +1,9 @@
- increment version in version.go
- increment version in snapcraft.yaml
- commit
- tag git with same version number
- make sure local build passes
- push tag to git
- 3.4.0, v3
- git push --tags
- make local xcompile (builds binaries for all platforms)
- git release
./scripts/release.sh
./scripts/upload.sh
- tag git with same version number
- commit vX tag - this will trigger github actions
- use github actions to publish docker and make github release
- snapcraft
- will auto create a candidate, test it works then promote

View File

@@ -1,22 +1,22 @@
#!/bin/bash
set -e
# This assumes that gonative and gox is installed as per the 'one time setup' instructions
# at https://github.com/inconshreveable/gonative
CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -output="build/yq_{{.OS}}_{{.Arch}}"
# include non-default linux builds too
CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -os=linux -output="build/yq_{{.OS}}_{{.Arch}}"
CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -output="build/yq_{{.OS}}_{{.Arch}}" --osarch="darwin/amd64 freebsd/386 freebsd/amd64 freebsd/arm linux/386 linux/amd64 linux/arm linux/arm64 linux/mips linux/mips64 linux/mips64le linux/mipsle linux/ppc64 linux/ppc64le linux/s390x netbsd/386 netbsd/amd64 netbsd/arm openbsd/386 openbsd/amd64 windows/386 windows/amd64"
cd build
rhash -r -a . -P -o checksums
rhash -r -a . -o checksums
rhash --list-hashes > checksums_hashes_order
find . | xargs -I {} tar czvf {}.tar.gz {}
find . -executable -type f | xargs -I {} tar czvf {}.tar.gz {}
# just in case find thinks this is executable...
rm -f checksums_hashes_order.tar.gz
rm -f checksums.tar.gz
rm checksums_hashes_order.tar.gz
rm checksums.tar.gz
rm yq_windows_386.exe.tar.gz
rm yq_windows_amd64.exe.tar.gz

View File

@@ -1,5 +1,5 @@
name: yq
version: '4.0.0'
version: '4.1.0'
summary: A lightweight and portable command-line YAML processor
description: |
The aim of the project is to be the jq or sed of yaml files.

View File

@@ -1,60 +0,0 @@
package main
// import (
// "fmt"
// "runtime"
// "testing"
// "github.com/mikefarah/yq/v2/pkg/marshal"
// "github.com/mikefarah/yq/v2/test"
// )
// func TestMultilineString(t *testing.T) {
// testString := `
// abcd
// efg`
// formattedResult, _ := marshal.NewYamlConverter().YamlToString(testString, false)
// test.AssertResult(t, testString, formattedResult)
// }
// func TestNewYaml(t *testing.T) {
// result, _ := newYaml([]string{"b.c", "3"})
// formattedResult := fmt.Sprintf("%v", result)
// test.AssertResult(t,
// "[{b [{c 3}]}]",
// formattedResult)
// }
// func TestNewYamlArray(t *testing.T) {
// result, _ := newYaml([]string{"[0].cat", "meow"})
// formattedResult := fmt.Sprintf("%v", result)
// test.AssertResult(t,
// "[[{cat meow}]]",
// formattedResult)
// }
// func TestNewYaml_WithScript(t *testing.T) {
// writeScript = "examples/instruction_sample.yaml"
// expectedResult := `b:
// c: cat
// e:
// - name: Mike Farah`
// result, _ := newYaml([]string{""})
// actualResult, _ := marshal.NewYamlConverter().YamlToString(result, true)
// test.AssertResult(t, expectedResult, actualResult)
// }
// func TestNewYaml_WithUnknownScript(t *testing.T) {
// writeScript = "fake-unknown"
// _, err := newYaml([]string{""})
// if err == nil {
// t.Error("Expected error due to unknown file")
// }
// var expectedOutput string
// if runtime.GOOS == "windows" {
// expectedOutput = `open fake-unknown: The system cannot find the file specified.`
// } else {
// expectedOutput = `open fake-unknown: no such file or directory`
// }
// test.AssertResult(t, expectedOutput, err.Error())
// }