Increment version

cmd/evaluate_all_command.go

@@ -17,6 +17,9 @@ func createEvaluateAllCommand() *cobra.Command {
 		Example: `
 # merges f2.yml into f1.yml (inplace)
 yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml		
+
+# use '-' as a filename to read from STDIN
+cat file2.yml | yq ea '.a.b' file1.yml - file3.yml
 `,
 		Long: "Evaluate All:\nUseful when you need to run an expression across several yaml documents or files. Consumes more memory than eval",
 		RunE: evaluateAll,

cmd/evalute_sequence_command.go

@@ -21,6 +21,9 @@ yq e '.a.b | length' f1.yml f2.yml
 # prints out the file
 yq e sample.yaml 
 
+# use '-' as a filename to read from STDIN
+cat file2.yml | yq e '.a.b' file1.yml - file3.yml
+
 # prints a new yaml document
 yq e -n '.a.b.c = "cat"' 
 

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.6.3"
+	Version = "4.7.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.6.3
+FROM mikefarah/yq:4.7.1
 
 COPY entrypoint.sh /entrypoint.sh
 

pkg/yqlib/candidate_node.go

@@ -8,9 +8,10 @@ import (
 )
 
 type CandidateNode struct {
-	Node      *yaml.Node    // the actual node
+	Node      *yaml.Node     // the actual node
-	Path      []interface{} /// the path we took to get to this node
+	Parent    *CandidateNode // parent node
-	Document  uint          // the document index of this node
+	Path      []interface{}  /// the path we took to get to this node
+	Document  uint           // the document index of this node
 	Filename  string
 	FileIndex int
 	// when performing op against all nodes given, this will treat all the nodes as one
@@ -31,6 +32,7 @@ func (n *CandidateNode) CreateChild(path interface{}, node *yaml.Node) *Candidat
 	return &CandidateNode{
 		Node:      node,
 		Path:      n.createChildPath(path),
+		Parent:    n,
 		Document:  n.Document,
 		Filename:  n.Filename,
 		FileIndex: n.FileIndex,

pkg/yqlib/doc/String Operators.md

@@ -20,6 +20,7 @@ cat; meow; 1; ; true
 
 ## Substitute / Replace string
 This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
+Note the use of `|=` to run in context of the current string value.
 
 Given a sample.yml file of:
 ```yaml
@@ -36,6 +37,7 @@ a: cats are great
 
 ## Substitute / Replace string with regex
 This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
+Note the use of `|=` to run in context of the current string value.
 
 Given a sample.yml file of:
 ```yaml
@@ -44,7 +46,7 @@ b: heat
 ```
 then
 ```bash
-yq eval '.[] |= sub("([a])", "${1}r")' sample.yml
+yq eval '.[] |= sub("(a)", "${1}r")' sample.yml
 ```
 will output
 ```yaml

pkg/yqlib/doc/aa.md

@@ -1,10 +1,44 @@
 # Operators
 
-In `yq` expressions are made up of operators. Operators have 0-2 arguments and run against the current 'matching' nodes in the expression tree.
+In `yq` expressions are made up of operators and pipes. A context of nodes is passed through the expression and each operation takes the context as input and returns a new context as output. That output is piped in as input for the next operation in the expression. To begin with, the context is set to the first yaml document of the first yaml file (if processing in sequence using eval).
 
 Lets look at a couple of examples.
 
-The `length` operator take no arguments, and will simply return the length of _each_ matching node. So if there were 2 nodes, one string and one array, length will update the 'matching' nodes context to be two new numeric scalar nodes representing the lengths of the orignal 'matching' nodes.
+## Example 1 - simple example
+
+Given a document like:
+
+```yaml
+- [a]
+- "cat"
+```
+
+with an expression:
+
+```
+.[] | length
+```
+
+`yq` will initially set the context as single node of the entire yaml document, an array of two elements.
+
+```yaml
+- [a]
+- "cat"
+```
+
+This gets piped into the splat operator `.[]` which will split out the context into a collection of two nodes `[a]` and `"cat"`. Note that this is _not_ a yaml array.
+
+The `length` operator take no arguments, and will simply return the length of _each_ matching node in the context. So for the context of `[a]` and `"cat"`, it will return a new context of `1` and `3`.
+
+This being the last operation in the expression, the results will be printed out:
+
+```
+1
+3
+```
+
+# Example 2 - operators with arguments.
+
 
 The `=` operator takes two arguments, a `lhs` expression and `rhs` expression. It runs the 'matching' nodes context against the `lhs` expression to find the nodes to update, lets call it `lhsNodes`, and then runs the matching nodes against the `rhs` to find the new values, lets call that `rhsNodes`. It updates the `lhsNodes` values with the `rhsNodes` values and _returns the original matching nodes_. This is important, where length changed the matching nodes to be new nodes with the length values, `=` returns the original matching nodes, albeit with some of the nodes values updated. So `.a = 3` will still return the parent matching node, but with the matching child updated.
 

pkg/yqlib/lib.go

@@ -95,7 +95,6 @@ var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs:
 var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator}
 var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator}
 var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
-var deleteImmediateChildOpType = &operationType{Type: "DELETE_IMMEDIATE_CHILD", NumArgs: 1, Precedence: 40, Handler: deleteImmediateChildOperator}
 
 type Operation struct {
 	OperationType *operationType

pkg/yqlib/operator_delete.go

@@ -18,48 +18,23 @@ func deleteChildOperator(d *dataTreeNavigator, context Context, expressionNode *
 	for el := nodesToDelete.MatchingNodes.Back(); el != nil; el = el.Prev() {
 		candidate := el.Value.(*CandidateNode)
 
-		if len(candidate.Path) > 0 {
+		//problem: context may already be '.a' and then I pass in '.a.a2'.
-			deleteImmediateChildOp := &Operation{
+		// should pass in .a2.
-				OperationType: deleteImmediateChildOpType,
+		if candidate.Parent == nil {
-				Value:         candidate.Path[len(candidate.Path)-1],
+			log.Info("Could not find parent of %v", candidate.GetKey())
-			}
+			return context, nil
-
-			deleteImmediateChildOpNode := &ExpressionNode{
-				Operation: deleteImmediateChildOp,
-				Rhs:       createTraversalTree(candidate.Path[0:len(candidate.Path)-1], traversePreferences{}, false),
-			}
-
-			_, err := d.GetMatchingNodes(contextToUse, deleteImmediateChildOpNode)
-			if err != nil {
-				return Context{}, err
-			}
 		}
-	}
-	return context, nil
-}
 
-func deleteImmediateChildOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+		parentNode := candidate.Parent.Node
-	parents, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		childPath := candidate.Path[len(candidate.Path)-1]
 
-	if err != nil {
-		return Context{}, err
-	}
-
-	childPath := expressionNode.Operation.Value
-
-	log.Debug("childPath to remove %v", childPath)
-
-	for el := parents.MatchingNodes.Front(); el != nil; el = el.Next() {
-		parent := el.Value.(*CandidateNode)
-		parentNode := unwrapDoc(parent.Node)
 		if parentNode.Kind == yaml.MappingNode {
-			deleteFromMap(parent, childPath)
+			deleteFromMap(candidate.Parent, childPath)
 		} else if parentNode.Kind == yaml.SequenceNode {
-			deleteFromArray(parent, childPath)
+			deleteFromArray(candidate.Parent, childPath)
 		} else {
 			return Context{}, fmt.Errorf("Cannot delete nodes from parent of tag %v", parentNode.Tag)
 		}
-
 	}
 	return context, nil
 }

pkg/yqlib/operator_delete_test.go

@@ -21,6 +21,70 @@ var deleteOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: {a2: frood}}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{a: {a1: fred, a2: frood}}`,
+		expression: `.a | del(.a1)`,
+		expected: []string{
+			"D0, P[a], (!!map)::{a2: frood}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: [1,2,3]`,
+		expression: `.a | del(.[1])`,
+		expected: []string{
+			"D0, P[a], (!!seq)::[1, 3]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[0, {a: cat, b: dog}]`,
+		expression: `.[1] | del(.a)`,
+		expected: []string{
+			"D0, P[1], (!!map)::{b: dog}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{a: cat, b: dog}]`,
+		expression: `.[0] | del(.a)`,
+		expected: []string{
+			"D0, P[0], (!!map)::{b: dog}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{a: {b: thing, c: frog}}]`,
+		expression: `.[0].a | del(.b)`,
+		expected: []string{
+			"D0, P[0 a], (!!map)::{c: frog}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{a: {b: thing, c: frog}}]`,
+		expression: `.[0] | del(.a.b)`,
+		expected: []string{
+			"D0, P[0], (!!map)::{a: {c: frog}}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [0, {b: thing, c: frog}]}`,
+		expression: `.a[1] | del(.b)`,
+		expected: []string{
+			"D0, P[a 1], (!!map)::{c: frog}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [0, {b: thing, c: frog}]}`,
+		expression: `.a | del(.[1].b)`,
+		expected: []string{
+			"D0, P[a], (!!seq)::[0, {c: frog}]\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `{a: {a1: fred, a2: frood}}`,

pkg/yqlib/operator_strings.go

@@ -62,7 +62,7 @@ func substituteStringOperator(d *dataTreeNavigator, context Context, expressionN
 		candidate := el.Value.(*CandidateNode)
 		node := unwrapDoc(candidate.Node)
 		if node.Tag != "!!str" {
-			return Context{}, fmt.Errorf("cannot sustitute with %v, can only substitute strings", node.Tag)
+			return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation.", node.Tag)
 		}
 
 		targetNode := substitute(node.Value, regEx, replacementText)

pkg/yqlib/operator_strings_test.go

@@ -15,7 +15,7 @@ var stringsOperatorScenarios = []expressionScenario{
 	},
 	{
 		description:    "Substitute / Replace string",
-		subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)",
+		subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)\nNote the use of `|=` to run in context of the current string value.",
 		document:       `a: dogs are great`,
 		expression:     `.a |= sub("dogs", "cats")`,
 		expected: []string{
@@ -24,9 +24,9 @@ var stringsOperatorScenarios = []expressionScenario{
 	},
 	{
 		description:    "Substitute / Replace string with regex",
-		subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)",
+		subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)\nNote the use of `|=` to run in context of the current string value.",
 		document:       "a: cat\nb: heat",
-		expression:     `.[] |= sub("([a])", "${1}r")`,
+		expression:     `.[] |= sub("(a)", "${1}r")`,
 		expected: []string{
 			"D0, P[], (doc)::a: cart\nb: heart\n",
 		},

release_instructions.txt

@@ -2,7 +2,7 @@
 - increment version in snapcraft.yaml
 - increment version in github-action/Dockerfile
 - make sure local build passes
-- run gosec (manual because docker platforms)
+- run ./scripts/secure.sh (manual because docker platforms)
 - commit version update changes
 - tag git with same version number
 - commit vX tag - this will trigger github actions

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.6.3'
+version: '4.7.1'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.