1
0
mirror of https://github.com/taigrr/yq synced 2025-01-18 04:53:17 -08:00

Compare commits

..

15 Commits

Author SHA1 Message Date
Mike Farah
f46c60f74d wip 2020-12-28 23:01:13 +11:00
Mike Farah
94a563dfd8 Updated docs 2020-12-28 11:57:20 +11:00
Mike Farah
0328cfd619 Added prettyPrint flag 2020-12-28 11:40:41 +11:00
Mike Farah
88663a6ce3 Added recurse keys operator 2020-12-28 11:24:42 +11:00
Mike Farah
b10a9ccfc6 Removed TraversePrefs 2020-12-28 10:29:43 +11:00
Mike Farah
9e9e15df73 More scenarios 2020-12-27 23:00:46 +11:00
Mike Farah
6cc6fdf322 Cleaning code 2020-12-27 22:56:15 +11:00
Mike Farah
a88c2dc5d3 Traverse Array Operator 2020-12-27 22:48:20 +11:00
Mike Farah
ea231006ed Refactoring traverse 2020-12-27 09:55:08 +11:00
Mike Farah
80f187f1a4 Refactoring traverse 2020-12-27 09:51:34 +11:00
Mike Farah
98e8b3479f Fixed nested array splat path 2020-12-25 12:49:05 +11:00
Mike Farah
eb539ff326 Fixed doc links 2020-12-23 10:37:51 +11:00
Mike Farah
c09f7aa707 Cleaning up docs 2020-12-23 10:30:13 +11:00
Mike Farah
7f5c380d16 v4.1.0 2020-12-23 10:23:54 +11:00
Mike Farah
6a05e517f1 Updated release instructions, remove gate for release 2020-12-23 10:23:09 +11:00
37 changed files with 907 additions and 306 deletions

View File

@@ -6,7 +6,6 @@ on:
jobs:
publishGitRelease:
environment: gitrelease
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2

View File

@@ -80,7 +80,7 @@ yq() {
### Go Get:
```
GO111MODULE=on go get github.com/mikefarah/yq/v4
GO111MODULE=on go get github.com/mikefarah/yq
```
## Community Supported Installation methods
@@ -126,7 +126,7 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
- Colorized yaml output
- [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/v/v4.x/traverse)
- [Sort yaml by keys](https://mikefarah.gitbook.io/yq/v/v4.x/sort-keys)
- Manipulate yaml [comments](https://app.gitbook.com/@mikefarah/s/yq/v/v4.x/comment-operators), [styling](https://app.gitbook.com/@mikefarah/s/yq/v/v4.x/style), [tags](https://app.gitbook.com/@mikefarah/s/yq/v/v4.x/tag) and [anchors](https://app.gitbook.com/@mikefarah/s/yq/v/v4.x/explode).
- Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/comment-operators), [styling](https://mikefarah.gitbook.io/yq/style), [tags](https://mikefarah.gitbook.io/yq/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/anchor-and-alias-operators).
- [Update yaml inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
- [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/v/v4.x/select#select-and-update-matching-values-in-map)
- Keeps yaml formatting and comments when updating (though there are issues with whitespace)
@@ -158,6 +158,7 @@ Flags:
-M, --no-colors force print with no colors
-N, --no-doc Don't print document separators (---)
-n, --null-input Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.
-P, --prettyPrint pretty print, shorthand for '... style = ""'
-j, --tojson output as json. Set indent to 0 to print json in one line.
-v, --verbose verbose mode
-V, --version Print version information and quit

View File

@@ -14,5 +14,6 @@ var noDocSeparators = false
var nullInput = false
var verbose = false
var version = false
var prettyPrint = false
var completedSuccessfully = false

View File

@@ -65,19 +65,19 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
switch len(args) {
case 0:
if pipingStdIn {
err = allAtOnceEvaluator.EvaluateFiles("", []string{"-"}, printer)
err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer)
} else {
cmd.Println(cmd.UsageString())
return nil
}
case 1:
if nullInput {
err = yqlib.NewStreamEvaluator().EvaluateNew(args[0], printer)
err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(args[0]), printer)
} else {
err = allAtOnceEvaluator.EvaluateFiles("", []string{args[0]}, printer)
err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer)
}
default:
err = allAtOnceEvaluator.EvaluateFiles(args[0], args[1:], printer)
err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer)
}
completedSuccessfully = err == nil

View File

@@ -33,6 +33,16 @@ yq e '.a.b = "cool"' -i file.yaml
}
return cmdEvalSequence
}
func processExpression(expression string) string {
if prettyPrint && expression == "" {
return `... style=""`
} else if prettyPrint {
return fmt.Sprintf("%v | ... style= \"\"", expression)
}
return expression
}
func evaluateSequence(cmd *cobra.Command, args []string) error {
cmd.SilenceUsage = true
// 0 args, read std in
@@ -76,16 +86,16 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
switch len(args) {
case 0:
if pipingStdIn {
err = streamEvaluator.EvaluateFiles("", []string{"-"}, printer)
err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer)
} else {
cmd.Println(cmd.UsageString())
return nil
}
case 1:
if nullInput {
err = streamEvaluator.EvaluateNew(args[0], printer)
err = streamEvaluator.EvaluateNew(processExpression(args[0]), printer)
} else {
err = streamEvaluator.EvaluateFiles("", []string{args[0]}, printer)
err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer)
}
default:
err = streamEvaluator.EvaluateFiles(args[0], args[1:], printer)

View File

@@ -48,6 +48,7 @@ func New() *cobra.Command {
rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace of first yaml file given.")
rootCmd.PersistentFlags().BoolVarP(&unwrapScalar, "unwrapScalar", "", true, "unwrap scalar, print the value with no quotes, colors or comments")
rootCmd.PersistentFlags().BoolVarP(&prettyPrint, "prettyPrint", "P", false, "pretty print, shorthand for '... style = \"\"'")
rootCmd.PersistentFlags().BoolVarP(&exitStatus, "exit-status", "e", false, "set exit status if there are no matches or null or false is returned")
rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")

View File

@@ -11,7 +11,7 @@ var (
GitDescribe string
// Version is main version number that is being run at the moment.
Version = "4.0.0"
Version = "4.2.0"
// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release

View File

@@ -21,6 +21,14 @@ func (n *CandidateNode) GetKey() string {
return fmt.Sprintf("%v - %v", n.Document, n.Path)
}
func (n *CandidateNode) CreateChildPath(path interface{}) []interface{} {
//don't use append as they may actually modify the path of the orignal node!
newPath := make([]interface{}, len(n.Path)+1)
copy(newPath, n.Path)
newPath[len(n.Path)] = path
return newPath
}
func (n *CandidateNode) Copy() (*CandidateNode, error) {
clone := &CandidateNode{}
err := copier.Copy(clone, n)

View File

@@ -112,7 +112,7 @@ a:
```
then
```bash
yq eval '(.a.[] | select(. == "apple")) = "frog"' sample.yml
yq eval '(.a[] | select(. == "apple")) = "frog"' sample.yml
```
will output
```yaml

38
pkg/yqlib/doc/Keys.md Normal file
View File

@@ -0,0 +1,38 @@
## Get Keys
Given a sample.yml file of:
```yaml
a:
b: cat
c: dog
d: frog
```
then
```bash
yq eval '.a | keys' sample.yml
```
will output
```yaml
3
```
## Set key style
Given a sample.yml file of:
```yaml
a:
b: cat
c: dog
d: frog
```
then
```bash
yq eval '(.a | keys) style = 'single'' sample.yml
```
will output
```yaml
a:
b: cat
c: dog
d: frog
```

View File

@@ -1,8 +1,55 @@
This operator recursively matches all children nodes given of a particular element, including that node itself. This is most often used to apply a filter recursively against all matches, for instance to set the `style` of all nodes in a yaml doc:
This operator recursively matches all children nodes given of a particular element, including that node itself. This is most often used to apply a filter recursively against all matches. It can be used in either the
## match values form `..`
This will, like the `jq` equivalent, recursively match all _value_ nodes. Use it to find/manipulate particular values.
For instance to set the `style` of all _value_ nodes in a yaml doc, excluding map keys:
```bash
yq eval '.. style= "flow"' file.yaml
```
## match values and map keys form `...`
The also includes map keys in the results set. This is particularly useful in YAML as unlike JSON, map keys can have their own styling, tags and use anchors and aliases.
For instance to set the `style` of all nodes in a yaml doc, including the map keys:
```bash
yq eval '... style= "flow"' file.yaml
```
## Recurse map (values only)
Given a sample.yml file of:
```yaml
a: frog
```
then
```bash
yq eval '..' sample.yml
```
will output
```yaml
a: frog
frog
```
## Recurse map (values and keys)
Note that the map key appears in the results
Given a sample.yml file of:
```yaml
a: frog
```
then
```bash
yq eval '...' sample.yml
```
will output
```yaml
a: frog
a
frog
```
## Aliases are not traversed
Given a sample.yml file of:
```yaml

View File

@@ -40,6 +40,26 @@ c: "3.2"
e: "true"
```
## Set double quote style on map keys too
Given a sample.yml file of:
```yaml
a: cat
b: 5
c: 3.2
e: true
```
then
```bash
yq eval '... style="double"' sample.yml
```
will output
```yaml
"a": "cat"
"b": "5"
"c": "3.2"
"e": "true"
```
## Set single quote style
Given a sample.yml file of:
```yaml
@@ -126,18 +146,18 @@ will output
```
## Pretty print
Set empty (default) quote style
Set empty (default) quote style, note the usage of `...` to match keys too.
Given a sample.yml file of:
```yaml
a: cat
b: 5
c: 3.2
e: true
"b": 5
'c': 3.2
"e": true
```
then
```bash
yq eval '.. style=""' sample.yml
yq eval '... style=""' sample.yml
```
will output
```yaml

View File

@@ -106,7 +106,7 @@ b: *cat
```
then
```bash
yq eval '.b.[]' sample.yml
yq eval '.b[]' sample.yml
```
will output
```yaml
@@ -290,7 +290,7 @@ foobar:
```
then
```bash
yq eval '.foobar.[]' sample.yml
yq eval '.foobar[]' sample.yml
```
will output
```yaml
@@ -356,7 +356,7 @@ foobar:
```
then
```bash
yq eval '.foobarList.[]' sample.yml
yq eval '.foobarList[]' sample.yml
```
will output
```yaml
@@ -366,3 +366,21 @@ bar_thing
foobarList_c
```
## Select multiple indices
Given a sample.yml file of:
```yaml
a:
- a
- b
- c
```
then
```bash
yq eval '.a[0, 2]' sample.yml
```
will output
```yaml
a
c
```

View File

@@ -1,5 +1,19 @@
This operator recursively matches all children nodes given of a particular element, including that node itself. This is most often used to apply a filter recursively against all matches, for instance to set the `style` of all nodes in a yaml doc:
This operator recursively matches all children nodes given of a particular element, including that node itself. This is most often used to apply a filter recursively against all matches. It can be used in either the
## match values form `..`
This will, like the `jq` equivalent, recursively match all _value_ nodes. Use it to find/manipulate particular values.
For instance to set the `style` of all _value_ nodes in a yaml doc, excluding map keys:
```bash
yq eval '.. style= "flow"' file.yaml
```
## match values and map keys form `...`
The also includes map keys in the results set. This is particularly useful in YAML as unlike JSON, map keys can have their own styling, tags and use anchors and aliases.
For instance to set the `style` of all nodes in a yaml doc, including the map keys:
```bash
yq eval '... style= "flow"' file.yaml
```

View File

@@ -19,7 +19,7 @@ type OperationType struct {
}
// operators TODO:
// - cookbook doc for common things
// - keys operator for controlling key metadata (particularly anchors/aliases)
// - mergeEmpty (sets only if the document is empty, do I do that now?)
var Or = &OperationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: OrOperator}
@@ -49,6 +49,7 @@ var CreateMap = &OperationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 40, H
var ShortPipe = &OperationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: 45, Handler: PipeOperator}
var Length = &OperationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: LengthOperator}
var Keys = &OperationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: KeysOperator}
var Collect = &OperationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: CollectOperator}
var GetStyle = &OperationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: GetStyleOperator}
var GetTag = &OperationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: GetTagOperator}
@@ -65,6 +66,7 @@ var SortKeys = &OperationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Han
var CollectObject = &OperationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: CollectObjectOperator}
var TraversePath = &OperationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
var TraverseArray = &OperationType{Type: "TRAVERSE_ARRAY", NumArgs: 1, Precedence: 50, Handler: TraverseArrayOperator}
var DocumentFilter = &OperationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: SelfOperator}

View File

@@ -118,5 +118,5 @@ func TestAnchorAliaseOperatorScenarios(t *testing.T) {
for _, tt := range anchorOperatorScenarios {
testScenario(t, &tt)
}
documentScenarios(t, "Anchor and Aliases Operators", anchorOperatorScenarios)
documentScenarios(t, "Anchor and Alias Operators", anchorOperatorScenarios)
}

View File

@@ -80,7 +80,15 @@ var assignOperatorScenarios = []expressionScenario{
{
description: "Update selected results",
document: `{a: {b: apple, c: cactus}}`,
expression: `(.a.[] | select(. == "apple")) = "frog"`,
expression: `(.a[] | select(. == "apple")) = "frog"`,
expected: []string{
"D0, P[], (doc)::{a: {b: frog, c: cactus}}\n",
},
},
{
skipDoc: true,
document: `{a: {b: apple, c: cactus}}`,
expression: `(.a.[] | select(. == "apple")) = "frog"`,
expected: []string{
"D0, P[], (doc)::{a: {b: frog, c: cactus}}\n",
},

View File

@@ -55,7 +55,7 @@ func CollectObjectOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *
func createChildCandidate(candidate *CandidateNode, index int) *CandidateNode {
return &CandidateNode{
Document: candidate.Document,
Path: append(candidate.Path, index),
Path: candidate.CreateChildPath(index),
Filename: candidate.Filename,
Node: candidate.Node.Content[index],
}
@@ -67,7 +67,9 @@ func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.
}
candidate := remainingMatches.Remove(remainingMatches.Front()).(*CandidateNode)
splatted, err := Splat(d, nodeToMap(candidate))
splatted, err := Splat(d, nodeToMap(candidate),
&TraversePreferences{FollowAlias: false, IncludeMapKeys: false})
for splatEl := splatted.Front(); splatEl != nil; splatEl = splatEl.Next() {
splatEl.Value.(*CandidateNode).Path = nil

View File

@@ -75,7 +75,7 @@ func deleteFromMap(candidate *CandidateNode, childPath interface{}) {
childCandidate := &CandidateNode{
Node: value,
Document: candidate.Document,
Path: append(candidate.Path, key.Value),
Path: candidate.CreateChildPath(key.Value),
}
shouldDelete := key.Value == childPath

View File

@@ -0,0 +1,34 @@
package yqlib
import (
"container/list"
"fmt"
yaml "gopkg.in/yaml.v3"
)
func KeysOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
log.Debugf("-- KeyOperation")
var results = list.New()
for el := matchMap.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
targetNode := UnwrapDoc(candidate.Node)
switch targetNode.Kind {
case yaml.MappingNode:
node := &yaml.Node{Kind: yaml.SequenceNode, Value: fmt.Sprintf("%v", length), Tag: "!!int"}
case yaml.SequenceNode:
length = len(targetNode.Content)
default:
length = 0
}
node := &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", length), Tag: "!!int"}
lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
results.PushBack(lengthCand)
}
return results, nil
}

View File

@@ -0,0 +1,88 @@
package yqlib
import (
"testing"
)
var keyOperatorScenarios = []expressionScenario{
{
description: "Get Keys of map",
document: `{a: {b: cat, c: dog, d: frog}}`,
expression: `.a | keys`,
expected: []string{
"D0, P[a], (!!seq)::- b\n- c\n- d\n",
},
},
{
skipDoc: true,
document: `{a1: {b: cat, c: dog, d: frog}, a2: {e: cat, f: dog, g: frog}}`,
expression: `(.a1, .a2) | keys`,
expected: []string{
"D0, P[a1], (!!seq)::- b\n- c\n- d\n",
"D0, P[a2], (!!seq)::- e\n- f\n- g\n",
},
},
{
description: "Get Keys of array",
document: `{a: [0,1,2]}`,
expression: `.a | keys`,
expected: []string{
"D0, P[a], (!!seq)::0\n",
"D0, P[a], (!!int)::1\n",
"D0, P[a], (!!int)::2\n",
},
},
{
description: "Set key style",
document: `{a: {b: cat, c: dog, d: frog}}`,
expression: `(.a | keys | ..) style = 'single'`,
expected: []string{
"D0, P[], (!!doc)::{a: {'b': cat, 'c': dog, 'd': frog}}\n",
},
},
{
description: "Set key alias",
document: `{a: {b: cat, c: dog, d: frog}}`,
expression: `(.a | keys | .. | select(.=="b")) alias = 'boo'`,
expected: []string{
"D0, P[], (!!doc)::{a: {*meow: cat, c: dog, d: frog}}\n",
},
},
{
description: "Set key alias",
document: `{a: {b: cat, c: dog, d: frog}}`,
expression: `(.a | key("b")) alias = 'boo'`,
expected: []string{
"D0, P[], (!!doc)::{a: {*meow: cat, c: dog, d: frog}}\n",
},
},
{
skipDoc: true,
document: `{a: [0,1,2]}`,
expression: `(.a | keys) style = 'single'`,
expected: []string{
"D0, P[a], (!!int)::0\n",
"D0, P[a], (!!int)::1\n",
"D0, P[a], (!!int)::2\n",
},
},
{
skipDoc: true,
document: `a: cat`,
expression: `.a | keys`,
expected: []string{},
},
{
skipDoc: true,
document: `{}`,
expression: `.a | keys`,
expected: []string{},
},
}
func TestKeyOperatorScenarios(t *testing.T) {
for _, tt := range keyOperatorScenarios {
testScenario(t, &tt)
}
documentScenarios(t, "Keys", keyOperatorScenarios)
}

View File

@@ -85,7 +85,9 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode,
var results = list.New()
// shouldn't recurse arrays if appending
err := recursiveDecent(d, results, nodeToMap(rhs), !shouldAppendArrays)
prefs := &RecursiveDescentPreferences{RecurseArray: !shouldAppendArrays,
TraversePreferences: &TraversePreferences{FollowAlias: false}}
err := recursiveDecent(d, results, nodeToMap(rhs), prefs)
if err != nil {
return nil, err
}

View File

@@ -13,6 +13,23 @@ var pathOperatorScenarios = []expressionScenario{
"D0, P[a b], (!!seq)::- a\n- b\n",
},
},
{
skipDoc: true,
document: `a:
b:
c:
- 0
- 1
- 2
- 3`,
expression: `.a.b.c.[]`,
expected: []string{
"D0, P[a b c 0], (!!int)::0\n",
"D0, P[a b c 1], (!!int)::1\n",
"D0, P[a b c 2], (!!int)::2\n",
"D0, P[a b c 3], (!!int)::3\n",
},
},
{
description: "Get map key",
document: `{a: {b: cat}}`,

View File

@@ -6,10 +6,16 @@ import (
yaml "gopkg.in/yaml.v3"
)
type RecursiveDescentPreferences struct {
TraversePreferences *TraversePreferences
RecurseArray bool
}
func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
var results = list.New()
err := recursiveDecent(d, results, matchMap, true)
preferences := pathNode.Operation.Preferences.(*RecursiveDescentPreferences)
err := recursiveDecent(d, results, matchMap, preferences)
if err != nil {
return nil, err
}
@@ -17,7 +23,7 @@ func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNod
return results, nil
}
func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List, recurseArray bool) error {
func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List, preferences *RecursiveDescentPreferences) error {
for el := matchMap.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
@@ -27,14 +33,14 @@ func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.Li
results.PushBack(candidate)
if candidate.Node.Kind != yaml.AliasNode && len(candidate.Node.Content) > 0 &&
(recurseArray || candidate.Node.Kind != yaml.SequenceNode) {
(preferences.RecurseArray || candidate.Node.Kind != yaml.SequenceNode) {
children, err := Splat(d, nodeToMap(candidate))
children, err := Splat(d, nodeToMap(candidate), preferences.TraversePreferences)
if err != nil {
return err
}
err = recursiveDecent(d, results, children, recurseArray)
err = recursiveDecent(d, results, children, preferences)
if err != nil {
return err
}

View File

@@ -13,6 +13,14 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
"D0, P[], (!!map)::{}\n",
},
},
{
skipDoc: true,
document: `{}`,
expression: `...`,
expected: []string{
"D0, P[], (!!map)::{}\n",
},
},
{
skipDoc: true,
document: `[]`,
@@ -21,6 +29,14 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
"D0, P[], (!!seq)::[]\n",
},
},
{
skipDoc: true,
document: `[]`,
expression: `...`,
expected: []string{
"D0, P[], (!!seq)::[]\n",
},
},
{
skipDoc: true,
document: `cat`,
@@ -31,13 +47,32 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
},
{
skipDoc: true,
document: `{a: frog}`,
expression: `..`,
document: `cat`,
expression: `...`,
expected: []string{
"D0, P[], (!!str)::cat\n",
},
},
{
description: "Recurse map (values only)",
document: `{a: frog}`,
expression: `..`,
expected: []string{
"D0, P[], (!!map)::{a: frog}\n",
"D0, P[a], (!!str)::frog\n",
},
},
{
description: "Recurse map (values and keys)",
subdescription: "Note that the map key appears in the results",
document: `{a: frog}`,
expression: `...`,
expected: []string{
"D0, P[], (!!map)::{a: frog}\n",
"D0, P[a], (!!str)::a\n",
"D0, P[a], (!!str)::frog\n",
},
},
{
skipDoc: true,
document: `{a: {b: apple}}`,
@@ -48,6 +83,18 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
"D0, P[a b], (!!str)::apple\n",
},
},
{
skipDoc: true,
document: `{a: {b: apple}}`,
expression: `...`,
expected: []string{
"D0, P[], (!!map)::{a: {b: apple}}\n",
"D0, P[a], (!!str)::a\n",
"D0, P[a], (!!map)::{b: apple}\n",
"D0, P[a b], (!!str)::b\n",
"D0, P[a b], (!!str)::apple\n",
},
},
{
skipDoc: true,
document: `[1,2,3]`,
@@ -59,6 +106,17 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
"D0, P[2], (!!int)::3\n",
},
},
{
skipDoc: true,
document: `[1,2,3]`,
expression: `...`,
expected: []string{
"D0, P[], (!!seq)::[1, 2, 3]\n",
"D0, P[0], (!!int)::1\n",
"D0, P[1], (!!int)::2\n",
"D0, P[2], (!!int)::3\n",
},
},
{
skipDoc: true,
document: `[{a: cat},2,true]`,
@@ -71,6 +129,19 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
"D0, P[2], (!!bool)::true\n",
},
},
{
skipDoc: true,
document: `[{a: cat},2,true]`,
expression: `...`,
expected: []string{
"D0, P[], (!!seq)::[{a: cat}, 2, true]\n",
"D0, P[0], (!!map)::{a: cat}\n",
"D0, P[0 a], (!!str)::a\n",
"D0, P[0 a], (!!str)::cat\n",
"D0, P[1], (!!int)::2\n",
"D0, P[2], (!!bool)::true\n",
},
},
{
description: "Aliases are not traversed",
document: `{a: &cat {c: frog}, b: *cat}`,
@@ -79,6 +150,20 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
"D0, P[a], (!!seq)::- {a: &cat {c: frog}, b: *cat}\n- &cat {c: frog}\n- frog\n- *cat\n",
},
},
{
skipDoc: true,
document: `{a: &cat {c: frog}, b: *cat}`,
expression: `...`,
expected: []string{
"D0, P[], (!!map)::{a: &cat {c: frog}, b: *cat}\n",
"D0, P[a], (!!str)::a\n",
"D0, P[a], (!!map)::&cat {c: frog}\n",
"D0, P[a c], (!!str)::c\n",
"D0, P[a c], (!!str)::frog\n",
"D0, P[b], (!!str)::b\n",
"D0, P[b], (alias)::*cat\n",
},
},
{
description: "Merge docs are not traversed",
document: mergeDocSample,
@@ -87,6 +172,14 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
"D0, P[foobar], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n",
},
},
{
skipDoc: true,
document: mergeDocSample,
expression: `.foobar | [...]`,
expected: []string{
"D0, P[foobar], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- c\n- foobar_c\n- !!merge <<\n- *foo\n- thing\n- foobar_thing\n",
},
},
{
skipDoc: true,
document: mergeDocSample,
@@ -100,6 +193,22 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
"D0, P[foobarList c], (!!str)::foobarList_c\n",
},
},
{
skipDoc: true,
document: mergeDocSample,
expression: `.foobarList | ...`,
expected: []string{
"D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n",
"D0, P[foobarList b], (!!str)::b\n",
"D0, P[foobarList b], (!!str)::foobarList_b\n",
"D0, P[foobarList <<], (!!merge)::!!merge <<\n",
"D0, P[foobarList <<], (!!seq)::[*foo, *bar]\n",
"D0, P[foobarList << 0], (alias)::*foo\n",
"D0, P[foobarList << 1], (alias)::*bar\n",
"D0, P[foobarList c], (!!str)::c\n",
"D0, P[foobarList c], (!!str)::foobarList_c\n",
},
},
}
func TestRecursiveDescentOperatorScenarios(t *testing.T) {

View File

@@ -21,6 +21,22 @@ var styleOperatorScenarios = []expressionScenario{
"D0, P[], (!!map)::a: \"cat\"\nb: \"5\"\nc: \"3.2\"\ne: \"true\"\n",
},
},
{
description: "Set double quote style on map keys too",
document: `{a: cat, b: 5, c: 3.2, e: true}`,
expression: `... style="double"`,
expected: []string{
"D0, P[], (!!map)::\"a\": \"cat\"\n\"b\": \"5\"\n\"c\": \"3.2\"\n\"e\": \"true\"\n",
},
},
{
skipDoc: true,
document: "bing: &foo frog\na:\n c: cat\n <<: [*foo]",
expression: `(... | select(tag=="!!str")) style="single"`,
expected: []string{
"D0, P[], (!!map)::'bing': &foo 'frog'\n'a':\n 'c': 'cat'\n !!merge <<: [*foo]\n",
},
},
{
description: "Set single quote style",
document: `{a: cat, b: 5, c: 3.2, e: true}`,
@@ -71,9 +87,9 @@ e: >-
},
{
description: "Pretty print",
subdescription: "Set empty (default) quote style",
document: `{a: cat, b: 5, c: 3.2, e: true}`,
expression: `.. style=""`,
subdescription: "Set empty (default) quote style, note the usage of `...` to match keys too. Note that there is a `--prettyPrint/-P` short flag for this.",
document: `{a: cat, "b": 5, 'c': 3.2, "e": true}`,
expression: `... style=""`,
expected: []string{
"D0, P[], (!!map)::a: cat\nb: 5\nc: 3.2\ne: true\n",
},

View File

@@ -1,46 +1,39 @@
package yqlib
import (
"fmt"
"container/list"
"fmt"
"strconv"
"github.com/elliotchance/orderedmap"
"gopkg.in/yaml.v3"
yaml "gopkg.in/yaml.v3"
)
type TraversePreferences struct {
DontFollowAlias bool
FollowAlias bool
IncludeMapKeys bool
}
func Splat(d *dataTreeNavigator, matches *list.List) (*list.List, error) {
preferences := &TraversePreferences{DontFollowAlias: true}
splatOperation := &Operation{OperationType: TraversePath, Value: "[]", Preferences: preferences}
splatTreeNode := &PathTreeNode{Operation: splatOperation}
return TraversePathOperator(d, matches, splatTreeNode)
func Splat(d *dataTreeNavigator, matches *list.List, prefs *TraversePreferences) (*list.List, error) {
return traverseNodesWithArrayIndices(matches, make([]*yaml.Node, 0), prefs)
}
func TraversePathOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
log.Debugf("-- Traversing")
var matchingNodeMap = list.New()
var newNodes []*CandidateNode
var err error
for el := matchMap.Front(); el != nil; el = el.Next() {
newNodes, err = traverse(d, el.Value.(*CandidateNode), pathNode.Operation)
newNodes, err := traverse(d, el.Value.(*CandidateNode), pathNode.Operation)
if err != nil {
return nil, err
}
for _, n := range newNodes {
matchingNodeMap.PushBack(n)
}
matchingNodeMap.PushBackList(newNodes)
}
return matchingNodeMap, nil
}
func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Operation) ([]*CandidateNode, error) {
func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Operation) (*list.List, error) {
log.Debug("Traversing %v", NodeToString(matchingNode))
value := matchingNode.Node
@@ -61,34 +54,8 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Oper
switch value.Kind {
case yaml.MappingNode:
log.Debug("its a map with %v entries", len(value.Content)/2)
var newMatches = orderedmap.NewOrderedMap()
err := traverseMap(newMatches, matchingNode, operation)
if err != nil {
return nil, err
}
if newMatches.Len() == 0 {
//no matches, create one automagically
valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}
node := matchingNode.Node
node.Content = append(node.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: operation.StringValue}, valueNode)
candidateNode := &CandidateNode{
Node: valueNode,
Path: append(matchingNode.Path, operation.StringValue),
Document: matchingNode.Document,
}
newMatches.Set(candidateNode.GetKey(), candidateNode)
}
arrayMatches := make([]*CandidateNode, newMatches.Len())
i := 0
for el := newMatches.Front(); el != nil; el = el.Next() {
arrayMatches[i] = el.Value.(*CandidateNode)
i++
}
return arrayMatches, nil
prefs := &TraversePreferences{FollowAlias: true}
return traverseMap(matchingNode, operation.StringValue, prefs, false)
case yaml.SequenceNode:
log.Debug("its a sequence of %v things!", len(value.Content))
@@ -101,105 +68,118 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Oper
case yaml.DocumentNode:
log.Debug("digging into doc node")
return traverse(d, &CandidateNode{
Node: matchingNode.Node.Content[0],
Document: matchingNode.Document}, operation)
Node: matchingNode.Node.Content[0],
Filename: matchingNode.Filename,
FileIndex: matchingNode.FileIndex,
Document: matchingNode.Document}, operation)
default:
return nil, nil
return list.New(), nil
}
}
func keyMatches(key *yaml.Node, pathNode *Operation) bool {
return pathNode.Value == "[]" || Match(key.Value, pathNode.StringValue)
}
func TraverseArrayOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
// rhs is a collect expression that will yield indexes to retreive of the arrays
func traverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, operation *Operation) error {
// value.Content is a concatenated array of key, value,
// so keys are in the even indexes, values in odd.
// merge aliases are defined first, but we only want to traverse them
// if we don't find a match directly on this node first.
//TODO ALIASES, auto creation?
node := candidate.Node
followAlias := true
if operation.Preferences != nil {
followAlias = !operation.Preferences.(*TraversePreferences).DontFollowAlias
rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
if err != nil {
return nil, err
}
var contents = node.Content
for index := 0; index < len(contents); index = index + 2 {
key := contents[index]
value := contents[index+1]
var indicesToTraverse = rhs.Front().Value.(*CandidateNode).Node.Content
prefs := &TraversePreferences{FollowAlias: true}
return traverseNodesWithArrayIndices(matchingNodes, indicesToTraverse, prefs)
}
log.Debug("checking %v (%v)", key.Value, key.Tag)
//skip the 'merge' tag, find a direct match first
if key.Tag == "!!merge" && followAlias {
log.Debug("Merge anchor")
err := traverseMergeAnchor(newMatches, candidate, value, operation)
if err != nil {
return err
}
} else if keyMatches(key, operation) {
log.Debug("MATCHED")
candidateNode := &CandidateNode{
Node: value,
Path: append(candidate.Path, key.Value),
Document: candidate.Document,
}
newMatches.Set(candidateNode.GetKey(), candidateNode)
func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, prefs *TraversePreferences) (*list.List, error) {
var matchingNodeMap = list.New()
for el := matchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
newNodes, err := traverseArrayIndices(candidate, indicesToTraverse, prefs)
if err != nil {
return nil, err
}
matchingNodeMap.PushBackList(newNodes)
}
return nil
return matchingNodeMap, nil
}
func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, operation *Operation) error {
switch value.Kind {
case yaml.AliasNode:
candidateNode := &CandidateNode{
Node: value.Alias,
Path: originalCandidate.Path,
Document: originalCandidate.Document,
}
return traverseMap(newMatches, candidateNode, operation)
case yaml.SequenceNode:
for _, childValue := range value.Content {
err := traverseMergeAnchor(newMatches, originalCandidate, childValue, operation)
if err != nil {
return err
}
}
func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, prefs *TraversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
node := matchingNode.Node
if node.Tag == "!!null" {
log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
// auto vivification, make it into an empty array
node.Tag = ""
node.Kind = yaml.SequenceNode
}
return nil
if node.Kind == yaml.AliasNode {
matchingNode.Node = node.Alias
return traverseArrayIndices(matchingNode, indicesToTraverse, prefs)
} else if node.Kind == yaml.SequenceNode {
return traverseArrayWithIndices(matchingNode, indicesToTraverse)
} else if node.Kind == yaml.MappingNode {
return traverseMapWithIndices(matchingNode, indicesToTraverse, prefs)
} else if node.Kind == yaml.DocumentNode {
return traverseArrayIndices(&CandidateNode{
Node: matchingNode.Node.Content[0],
Filename: matchingNode.Filename,
FileIndex: matchingNode.FileIndex,
Document: matchingNode.Document}, indicesToTraverse, prefs)
}
log.Debugf("OperatorArrayTraverse skipping %v as its a %v", matchingNode, node.Tag)
return list.New(), nil
}
func traverseArray(candidate *CandidateNode, operation *Operation) ([]*CandidateNode, error) {
log.Debug("operation Value %v", operation.Value)
if operation.Value == "[]" {
func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs *TraversePreferences) (*list.List, error) {
if len(indices) == 0 {
return traverseMap(candidate, "", prefs, true)
}
var contents = candidate.Node.Content
var newMatches = make([]*CandidateNode, len(contents))
var matchingNodeMap = list.New()
for _, indexNode := range indices {
log.Debug("traverseMapWithIndices: %v", indexNode.Value)
newNodes, err := traverseMap(candidate, indexNode.Value, prefs, false)
if err != nil {
return nil, err
}
matchingNodeMap.PushBackList(newNodes)
}
return matchingNodeMap, nil
}
func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*list.List, error) {
log.Debug("traverseArrayWithIndices")
var newMatches = list.New()
node := UnwrapDoc(candidate.Node)
if len(indices) == 0 {
log.Debug("splatting")
var index int64
for index = 0; index < int64(len(contents)); index = index + 1 {
newMatches[index] = &CandidateNode{
for index = 0; index < int64(len(node.Content)); index = index + 1 {
newMatches.PushBack(&CandidateNode{
Document: candidate.Document,
Path: append(candidate.Path, index),
Node: contents[index],
}
Path: candidate.CreateChildPath(index),
Node: node.Content[index],
})
}
return newMatches, nil
}
switch operation.Value.(type) {
case int64:
index := operation.Value.(int64)
for _, indexNode := range indices {
log.Debug("traverseArrayWithIndices: '%v'", indexNode.Value)
index, err := strconv.ParseInt(indexNode.Value, 10, 64)
if err != nil {
return nil, fmt.Errorf("Cannot index array with '%v' (%v)", indexNode.Value, err)
}
indexToUse := index
contentLength := int64(len(candidate.Node.Content))
contentLength := int64(len(node.Content))
for contentLength <= index {
candidate.Node.Content = append(candidate.Node.Content, &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"})
contentLength = int64(len(candidate.Node.Content))
node.Content = append(node.Content, &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"})
contentLength = int64(len(node.Content))
}
if indexToUse < 0 {
@@ -210,14 +190,115 @@ func traverseArray(candidate *CandidateNode, operation *Operation) ([]*Candidate
return nil, fmt.Errorf("Index [%v] out of range, array size is %v", index, contentLength)
}
return []*CandidateNode{&CandidateNode{
Node: candidate.Node.Content[indexToUse],
newMatches.PushBack(&CandidateNode{
Node: node.Content[indexToUse],
Document: candidate.Document,
Path: append(candidate.Path, index),
}}, nil
default:
log.Debug("argument not an int (%v), no array matches", operation.Value)
return nil, nil
Path: candidate.CreateChildPath(index),
})
}
return newMatches, nil
}
func keyMatches(key *yaml.Node, wantedKey string) bool {
return Match(key.Value, wantedKey)
}
func traverseMap(matchingNode *CandidateNode, key string, prefs *TraversePreferences, splat bool) (*list.List, error) {
var newMatches = orderedmap.NewOrderedMap()
err := doTraverseMap(newMatches, matchingNode, key, prefs, splat)
if err != nil {
return nil, err
}
if newMatches.Len() == 0 {
//no matches, create one automagically
valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}
node := matchingNode.Node
node.Content = append(node.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: key}, valueNode)
candidateNode := &CandidateNode{
Node: valueNode,
Path: append(matchingNode.Path, key),
Document: matchingNode.Document,
}
newMatches.Set(candidateNode.GetKey(), candidateNode)
}
results := list.New()
i := 0
for el := newMatches.Front(); el != nil; el = el.Next() {
results.PushBack(el.Value)
i++
}
return results, nil
}
func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, prefs *TraversePreferences, splat bool) error {
// value.Content is a concatenated array of key, value,
// so keys are in the even indexes, values in odd.
// merge aliases are defined first, but we only want to traverse them
// if we don't find a match directly on this node first.
node := candidate.Node
var contents = node.Content
for index := 0; index < len(contents); index = index + 2 {
key := contents[index]
value := contents[index+1]
log.Debug("checking %v (%v)", key.Value, key.Tag)
//skip the 'merge' tag, find a direct match first
if key.Tag == "!!merge" && prefs.FollowAlias {
log.Debug("Merge anchor")
err := traverseMergeAnchor(newMatches, candidate, value, wantedKey, prefs, splat)
if err != nil {
return err
}
} else if splat || keyMatches(key, wantedKey) {
log.Debug("MATCHED")
if prefs.IncludeMapKeys {
candidateNode := &CandidateNode{
Node: key,
Path: candidate.CreateChildPath(key.Value),
Document: candidate.Document,
}
newMatches.Set(fmt.Sprintf("keyOf-%v", candidateNode.GetKey()), candidateNode)
}
candidateNode := &CandidateNode{
Node: value,
Path: candidate.CreateChildPath(key.Value),
Document: candidate.Document,
}
newMatches.Set(candidateNode.GetKey(), candidateNode)
}
}
return nil
}
func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, prefs *TraversePreferences, splat bool) error {
switch value.Kind {
case yaml.AliasNode:
candidateNode := &CandidateNode{
Node: value.Alias,
Path: originalCandidate.Path,
Document: originalCandidate.Document,
}
return doTraverseMap(newMatches, candidateNode, wantedKey, prefs, splat)
case yaml.SequenceNode:
for _, childValue := range value.Content {
err := traverseMergeAnchor(newMatches, originalCandidate, childValue, wantedKey, prefs, splat)
if err != nil {
return err
}
}
}
return nil
}
func traverseArray(candidate *CandidateNode, operation *Operation) (*list.List, error) {
log.Debug("operation Value %v", operation.Value)
indices := []*yaml.Node{&yaml.Node{Value: operation.StringValue}}
return traverseArrayWithIndices(candidate, indices)
}

View File

@@ -65,7 +65,7 @@ var traversePathOperatorScenarios = []expressionScenario{
},
{
skipDoc: true,
document: `{}`,
document: ``,
expression: `.[1].a`,
expected: []string{
"D0, P[1 a], (!!null)::null\n",
@@ -137,7 +137,15 @@ var traversePathOperatorScenarios = []expressionScenario{
{
description: "Traversing aliases with splat",
document: `{a: &cat {c: frog}, b: *cat}`,
expression: `.b.[]`,
expression: `.b[]`,
expected: []string{
"D0, P[b c], (!!str)::frog\n",
},
},
{
skipDoc: true,
document: `{a: &cat {c: frog}, b: *cat}`,
expression: `.b.[]`,
expected: []string{
"D0, P[b c], (!!str)::frog\n",
},
@@ -150,12 +158,6 @@ var traversePathOperatorScenarios = []expressionScenario{
"D0, P[b c], (!!str)::frog\n",
},
},
{
skipDoc: true,
document: `[1,2,3]`,
expression: `.b`,
expected: []string{},
},
{
description: "Traversing arrays by index",
document: `[1,2,3]`,
@@ -215,7 +217,17 @@ var traversePathOperatorScenarios = []expressionScenario{
{
description: "Splatting merge anchors",
document: mergeDocSample,
expression: `.foobar.[]`,
expression: `.foobar[]`,
expected: []string{
"D0, P[foobar c], (!!str)::foo_c\n",
"D0, P[foobar a], (!!str)::foo_a\n",
"D0, P[foobar thing], (!!str)::foobar_thing\n",
},
},
{
skipDoc: true,
document: mergeDocSample,
expression: `.foobar.[]`,
expected: []string{
"D0, P[foobar c], (!!str)::foo_c\n",
"D0, P[foobar a], (!!str)::foo_a\n",
@@ -266,7 +278,7 @@ var traversePathOperatorScenarios = []expressionScenario{
{
description: "Splatting merge anchor lists",
document: mergeDocSample,
expression: `.foobarList.[]`,
expression: `.foobarList[]`,
expected: []string{
"D0, P[foobarList b], (!!str)::bar_b\n",
"D0, P[foobarList a], (!!str)::foo_a\n",
@@ -274,6 +286,123 @@ var traversePathOperatorScenarios = []expressionScenario{
"D0, P[foobarList c], (!!str)::foobarList_c\n",
},
},
{
skipDoc: true,
document: mergeDocSample,
expression: `.foobarList.[]`,
expected: []string{
"D0, P[foobarList b], (!!str)::bar_b\n",
"D0, P[foobarList a], (!!str)::foo_a\n",
"D0, P[foobarList thing], (!!str)::bar_thing\n",
"D0, P[foobarList c], (!!str)::foobarList_c\n",
},
},
{
skipDoc: true,
document: `[a,b,c]`,
expression: `.[]`,
expected: []string{
"D0, P[0], (!!str)::a\n",
"D0, P[1], (!!str)::b\n",
"D0, P[2], (!!str)::c\n",
},
},
{
skipDoc: true,
document: `[a,b,c]`,
expression: `[]`,
expected: []string{
"D0, P[], (!!seq)::[]\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a[0]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
},
},
{
description: "Select multiple indices",
document: `{a: [a,b,c]}`,
expression: `.a[0, 2]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
"D0, P[a 2], (!!str)::c\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a.[0, 2]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
"D0, P[a 2], (!!str)::c\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a[-1]`,
expected: []string{
"D0, P[a -1], (!!str)::c\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a.[-1]`,
expected: []string{
"D0, P[a -1], (!!str)::c\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a[-2]`,
expected: []string{
"D0, P[a -2], (!!str)::b\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a.[-2]`,
expected: []string{
"D0, P[a -2], (!!str)::b\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a[]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
"D0, P[a 1], (!!str)::b\n",
"D0, P[a 2], (!!str)::c\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a.[]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
"D0, P[a 1], (!!str)::b\n",
"D0, P[a 2], (!!str)::c\n",
},
},
{
skipDoc: true,
document: `{a: [a,b,c]}`,
expression: `.a | .[]`,
expected: []string{
"D0, P[a 0], (!!str)::a\n",
"D0, P[a 1], (!!str)::b\n",
"D0, P[a 2], (!!str)::c\n",
},
},
}
func TestTraversePathOperatorScenarios(t *testing.T) {

View File

@@ -38,7 +38,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
if s.document != "" {
inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml", 0)
if err != nil {
t.Error(err, s.document)
t.Error(err, s.document, s.expression)
return
}
} else {
@@ -55,7 +55,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
results, err = treeNavigator.GetMatchingNodes(inputs, node)
if err != nil {
t.Error(err)
t.Error(fmt.Errorf("%v: %v", err, s.expression))
return
}
test.AssertResultComplexWithContext(t, s.expected, resultsToString(results), fmt.Sprintf("exp: %v\ndoc: %v", s.expression, s.document))
@@ -167,17 +167,17 @@ func documentScenarios(t *testing.T, title string, scenarios []expressionScenari
if s.document != "" {
node, err := treeCreator.ParsePath(s.expression)
if err != nil {
t.Error(err)
t.Error(err, s.expression)
}
err = streamEvaluator.Evaluate("sample.yaml", strings.NewReader(formattedDoc), node, printer)
if err != nil {
t.Error(err)
t.Error(err, s.expression)
}
} else {
err = streamEvaluator.EvaluateNew(s.expression, printer)
if err != nil {
t.Error(err)
t.Error(err, s.expression)
}
}

View File

@@ -17,20 +17,35 @@ var pathTests = []struct {
append(make([]interface{}, 0), "[", "]"),
append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE"),
},
{
`.[]`,
append(make([]interface{}, 0), "TRAVERSE_ARRAY", "[", "]"),
append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
},
{
`.a[]`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "[]"),
append(make([]interface{}, 0), "a", "[]", "SHORT_PIPE"),
append(make([]interface{}, 0), "a", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "]"),
append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
},
{
`.a.[]`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "[]"),
append(make([]interface{}, 0), "a", "[]", "SHORT_PIPE"),
append(make([]interface{}, 0), "a", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "]"),
append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
},
{
`.a[0]`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
},
{
`.a.[0]`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
},
{
`.a[].c`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "[]", "SHORT_PIPE", "c"),
append(make([]interface{}, 0), "a", "[]", "SHORT_PIPE", "c", "SHORT_PIPE"),
append(make([]interface{}, 0), "a", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "]", "SHORT_PIPE", "c"),
append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "c", "SHORT_PIPE"),
},
{
`[3]`,
@@ -44,18 +59,18 @@ var pathTests = []struct {
},
{
`.a | .[].b == "apple"`,
append(make([]interface{}, 0), "a", "PIPE", "[]", "SHORT_PIPE", "b", "EQUALS", "apple (string)"),
append(make([]interface{}, 0), "a", "[]", "b", "SHORT_PIPE", "apple (string)", "EQUALS", "PIPE"),
append(make([]interface{}, 0), "a", "PIPE", "TRAVERSE_ARRAY", "[", "]", "SHORT_PIPE", "b", "EQUALS", "apple (string)"),
append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "apple (string)", "EQUALS", "PIPE"),
},
{
`(.a | .[].b) == "apple"`,
append(make([]interface{}, 0), "(", "a", "PIPE", "[]", "SHORT_PIPE", "b", ")", "EQUALS", "apple (string)"),
append(make([]interface{}, 0), "a", "[]", "b", "SHORT_PIPE", "PIPE", "apple (string)", "EQUALS"),
append(make([]interface{}, 0), "(", "a", "PIPE", "TRAVERSE_ARRAY", "[", "]", "SHORT_PIPE", "b", ")", "EQUALS", "apple (string)"),
append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "PIPE", "apple (string)", "EQUALS"),
},
{
`.[] | select(. == "*at")`,
append(make([]interface{}, 0), "[]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at (string)", ")"),
append(make([]interface{}, 0), "[]", "SELF", "*at (string)", "EQUALS", "SELECT", "PIPE"),
append(make([]interface{}, 0), "TRAVERSE_ARRAY", "[", "]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at (string)", ")"),
append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SELF", "*at (string)", "EQUALS", "SELECT", "PIPE"),
},
{
`[true]`,
@@ -89,8 +104,8 @@ var pathTests = []struct {
},
{
`{.a: .c, .b.[]: .f.g.[]}`,
append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "SHORT_PIPE", "[]", "CREATE_MAP", "f", "SHORT_PIPE", "g", "SHORT_PIPE", "[]", "}"),
append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "[]", "SHORT_PIPE", "f", "g", "SHORT_PIPE", "[]", "SHORT_PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "SHORT_PIPE"),
append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "]", "CREATE_MAP", "f", "SHORT_PIPE", "g", "SHORT_PIPE", "TRAVERSE_ARRAY", "[", "]", "}"),
append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "f", "g", "SHORT_PIPE", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "SHORT_PIPE"),
},
{
`explode(.a.b)`,
@@ -122,7 +137,6 @@ var pathTests = []struct {
append(make([]interface{}, 0), "SELF", "ASSIGN_COMMENT", "str (string)"),
append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_COMMENT"),
},
{
`.a.b tag="!!str"`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "b", "ASSIGN_TAG", "!!str (string)"),

View File

@@ -22,7 +22,7 @@ const (
CloseCollect
OpenCollectObject
CloseCollectObject
SplatOrEmptyCollect
TraverseArrayCollect
)
type Token struct {
@@ -49,8 +49,9 @@ func (t *Token) toString() string {
return "{"
} else if t.TokenType == CloseCollectObject {
return "}"
} else if t.TokenType == SplatOrEmptyCollect {
return "[]?"
} else if t.TokenType == TraverseArrayCollect {
return ".["
} else {
return "NFI"
}
@@ -114,23 +115,6 @@ func unwrap(value string) string {
return value[1 : len(value)-1]
}
func arrayIndextoken(precedingDot bool) lex.Action {
return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
var numberString = string(m.Bytes)
startIndex := 1
if precedingDot {
startIndex = 2
}
numberString = numberString[startIndex : len(numberString)-1]
var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) // nolint
if errParsingInt != nil {
return nil, errParsingInt
}
op := &Operation{OperationType: TraversePath, Value: number, StringValue: numberString}
return &Token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil
}
}
func numberValue() lex.Action {
return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
var numberString = string(m.Bytes)
@@ -188,12 +172,17 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`\(`), literalToken(OpenBracket, false))
lexer.Add([]byte(`\)`), literalToken(CloseBracket, true))
lexer.Add([]byte(`\.\[\]`), pathToken(false))
lexer.Add([]byte(`\.\.`), opToken(RecursiveDescent))
lexer.Add([]byte(`\.\[`), literalToken(TraverseArrayCollect, false))
lexer.Add([]byte(`\.\.`), opTokenWithPrefs(RecursiveDescent, nil, &RecursiveDescentPreferences{RecurseArray: true,
TraversePreferences: &TraversePreferences{FollowAlias: false, IncludeMapKeys: false}}))
lexer.Add([]byte(`\.\.\.`), opTokenWithPrefs(RecursiveDescent, nil, &RecursiveDescentPreferences{RecurseArray: true,
TraversePreferences: &TraversePreferences{FollowAlias: false, IncludeMapKeys: true}}))
lexer.Add([]byte(`,`), opToken(Union))
lexer.Add([]byte(`:\s*`), opToken(CreateMap))
lexer.Add([]byte(`length`), opToken(Length))
lexer.Add([]byte(`keys`), opToken(Keys))
lexer.Add([]byte(`sortKeys`), opToken(SortKeys))
lexer.Add([]byte(`select`), opToken(Select))
lexer.Add([]byte(`has`), opToken(Has))
@@ -231,8 +220,6 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`\s*\|=\s*`), opTokenWithPrefs(Assign, nil, &AssignOpPreferences{true}))
lexer.Add([]byte(`\.\[-?[0-9]+\]`), arrayIndextoken(true))
lexer.Add([]byte("( |\t|\n|\r)+"), skip)
lexer.Add([]byte(`d[0-9]+`), documentToken())
@@ -254,8 +241,6 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`"[^"]*"`), stringValue(true))
lexer.Add([]byte(`\[\]`), literalToken(SplatOrEmptyCollect, true))
lexer.Add([]byte(`\[`), literalToken(OpenCollect, false))
lexer.Add([]byte(`\]`), literalToken(CloseCollect, true))
lexer.Add([]byte(`\{`), literalToken(OpenCollectObject, false))
@@ -324,24 +309,16 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) {
func (p *pathTokeniser) handleToken(tokens []*Token, index int, postProcessedTokens []*Token) (tokensAccum []*Token, skipNextToken bool) {
skipNextToken = false
token := tokens[index]
if token.TokenType == SplatOrEmptyCollect {
if index > 0 && tokens[index-1].TokenType == OperationToken &&
tokens[index-1].Operation.OperationType == TraversePath {
// must be a splat without a preceding dot , e.g. .a[]
// lets put a pipe in front of it, and convert it to a traverse "[]" token
pipeOp := &Operation{OperationType: ShortPipe, Value: "PIPE"}
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: pipeOp})
if token.TokenType == TraverseArrayCollect {
//need to put a traverse array then a collect token
// do this by adding traverse then converting token to collect
traverseOp := &Operation{OperationType: TraversePath, Value: "[]", StringValue: "[]"}
token = &Token{TokenType: OperationToken, Operation: traverseOp, CheckForPostTraverse: true}
op := &Operation{OperationType: TraverseArray, StringValue: "TRAVERSE_ARRAY"}
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
token = &Token{TokenType: OpenCollect}
} else {
// gotta be a collect empty array, we need to split this into two tokens
// one OpenCollect, the other CloseCollect
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OpenCollect})
token = &Token{TokenType: CloseCollect, CheckForPostTraverse: true}
}
}
if index != len(tokens)-1 && token.AssignOperation != nil &&
@@ -359,5 +336,21 @@ func (p *pathTokeniser) handleToken(tokens []*Token, index int, postProcessedTok
op := &Operation{OperationType: ShortPipe, Value: "PIPE"}
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
}
if index != len(tokens)-1 && token.CheckForPostTraverse &&
tokens[index+1].TokenType == OpenCollect {
op := &Operation{OperationType: ShortPipe, Value: "PIPE"}
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
op = &Operation{OperationType: TraverseArray}
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
}
if index != len(tokens)-1 && token.CheckForPostTraverse &&
tokens[index+1].TokenType == TraverseArrayCollect {
op := &Operation{OperationType: ShortPipe, Value: "PIPE"}
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
}
return postProcessedTokens, skipNextToken
}

10
pkg/yqlib/test.yml Normal file
View File

@@ -0,0 +1,10 @@
{
"a1": {
"b": "cat",
"c": "dog"
},
"a2": {
"d": "cat",
"e": "dog"
}
}

View File

@@ -1,16 +1,9 @@
- increment version in version.go
- increment version in snapcraft.yaml
- commit
- tag git with same version number
- make sure local build passes
- push tag to git
- 3.4.0, v3
- git push --tags
- make local xcompile (builds binaries for all platforms)
- git release
./scripts/release.sh
./scripts/upload.sh
- tag git with same version number
- commit vX tag - this will trigger github actions
- use github actions to publish docker and make github release
- snapcraft
- will auto create a candidate, test it works then promote

View File

@@ -1,5 +1,5 @@
name: yq
version: '4.0.0'
version: '4.2.0'
summary: A lightweight and portable command-line YAML processor
description: |
The aim of the project is to be the jq or sed of yaml files.

View File

@@ -1,60 +0,0 @@
package main
// import (
// "fmt"
// "runtime"
// "testing"
// "github.com/mikefarah/yq/v2/pkg/marshal"
// "github.com/mikefarah/yq/v2/test"
// )
// func TestMultilineString(t *testing.T) {
// testString := `
// abcd
// efg`
// formattedResult, _ := marshal.NewYamlConverter().YamlToString(testString, false)
// test.AssertResult(t, testString, formattedResult)
// }
// func TestNewYaml(t *testing.T) {
// result, _ := newYaml([]string{"b.c", "3"})
// formattedResult := fmt.Sprintf("%v", result)
// test.AssertResult(t,
// "[{b [{c 3}]}]",
// formattedResult)
// }
// func TestNewYamlArray(t *testing.T) {
// result, _ := newYaml([]string{"[0].cat", "meow"})
// formattedResult := fmt.Sprintf("%v", result)
// test.AssertResult(t,
// "[[{cat meow}]]",
// formattedResult)
// }
// func TestNewYaml_WithScript(t *testing.T) {
// writeScript = "examples/instruction_sample.yaml"
// expectedResult := `b:
// c: cat
// e:
// - name: Mike Farah`
// result, _ := newYaml([]string{""})
// actualResult, _ := marshal.NewYamlConverter().YamlToString(result, true)
// test.AssertResult(t, expectedResult, actualResult)
// }
// func TestNewYaml_WithUnknownScript(t *testing.T) {
// writeScript = "fake-unknown"
// _, err := newYaml([]string{""})
// if err == nil {
// t.Error("Expected error due to unknown file")
// }
// var expectedOutput string
// if runtime.GOOS == "windows" {
// expectedOutput = `open fake-unknown: The system cannot find the file specified.`
// } else {
// expectedOutput = `open fake-unknown: no such file or directory`
// }
// test.AssertResult(t, expectedOutput, err.Error())
// }