Added string substitute command

pkg/yqlib/doc/Anchor and Alias Operators.md

@@ -1,4 +1,4 @@
-Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference aliases and remove anchor names).
+Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
 
 `yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
 

pkg/yqlib/doc/String Operators.md

@@ -18,6 +18,40 @@ will output
 cat; meow; 1; ; true
 ```
 
+## Substitute / Replace string
+This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
+
+Given a sample.yml file of:
+```yaml
+a: dogs are great
+```
+then
+```bash
+yq eval '.a |= sub("dogs", "cats")' sample.yml
+```
+will output
+```yaml
+a: cats are great
+```
+
+## Substitute / Replace string with regex
+This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
+
+Given a sample.yml file of:
+```yaml
+a: cat
+b: heat
+```
+then
+```bash
+yq eval '.[] |= sub("([a])", "${1}r")' sample.yml
+```
+will output
+```yaml
+a: cart
+b: heart
+```
+
 ## Split strings
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/headers/Anchor and Alias Operators.md

@@ -1,4 +1,4 @@
-Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference aliases and remove anchor names).
+Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
 
 `yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
 

pkg/yqlib/expression_tokeniser.go

@@ -264,6 +264,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`splitDoc`), opToken(splitDocumentOpType))
 
 	lexer.Add([]byte(`join`), opToken(joinStringOpType))
+	lexer.Add([]byte(`sub`), opToken(subStringOpType))
+
 	lexer.Add([]byte(`split`), opToken(splitStringOpType))
 	lexer.Add([]byte(`keys`), opToken(keysOpType))
 

pkg/yqlib/lib.go

@@ -75,6 +75,7 @@ var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50,
 var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator}
 var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
 var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}
+var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator}
 var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
 
 var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}

pkg/yqlib/operator_add.go

@@ -39,7 +39,7 @@ func toNodes(candidate *CandidateNode) []*yaml.Node {
 func addOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("Add operator")
 
-	return crossFunction(d, context, expressionNode, add)
+	return crossFunction(d, context, expressionNode, add, false)
 }
 
 func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

pkg/yqlib/operator_alternative.go

@@ -1,14 +1,14 @@
 package yqlib
 
-// corssFunction no matches
-// can boolean use crossfunction
-
 func alternativeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- alternative")
-	return crossFunction(d, context, expressionNode, alternativeFunc)
+	return crossFunction(d, context, expressionNode, alternativeFunc, true)
 }
 
 func alternativeFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	if lhs == nil {
+		return rhs, nil
+	}
 	lhs.Node = unwrapDoc(lhs.Node)
 	rhs.Node = unwrapDoc(rhs.Node)
 	log.Debugf("Alternative LHS: %v", lhs.Node.Tag)

pkg/yqlib/operator_alternative_test.go

@@ -13,6 +13,14 @@ var alternativeOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!str)::bridge\n",
 		},
 	},
+	{
+		expression: `select(tag == "seq") // "cat"`,
+		skipDoc:    true,
+		document:   `a: frog`,
+		expected: []string{
+			"D0, P[], (!!str)::cat\n",
+		},
+	},
 	{
 		description: "LHS is not defined",
 		expression:  `.a // "hello"`,

pkg/yqlib/operator_booleans.go

@@ -49,7 +49,7 @@ func orOperator(d *dataTreeNavigator, context Context, expressionNode *Expressio
 	return crossFunction(d, context, expressionNode, performBoolOp(
 		func(b1 bool, b2 bool) bool {
 			return b1 || b2
-		}))
+		}), false)
 }
 
 func andOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@@ -57,7 +57,7 @@ func andOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	return crossFunction(d, context, expressionNode, performBoolOp(
 		func(b1 bool, b2 bool) bool {
 			return b1 && b2
-		}))
+		}), false)
 }
 
 func notOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {

pkg/yqlib/operator_create_map.go

@@ -62,7 +62,7 @@ func sequenceFor(d *dataTreeNavigator, context Context, matchingNode *CandidateN
 			}
 
 			return &CandidateNode{Node: &node, Document: document, Path: path}, nil
-		})
+		}, false)
 
 	if err != nil {
 		return nil, err

pkg/yqlib/operator_equals.go

@@ -4,7 +4,7 @@ import "gopkg.in/yaml.v3"
 
 func equalsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- equalsOperation")
-	return crossFunction(d, context, expressionNode, isEquals(false))
+	return crossFunction(d, context, expressionNode, isEquals(false), false)
 }
 
 func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
@@ -29,5 +29,5 @@ func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *Candid
 
 func notEqualsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- equalsOperation")
-	return crossFunction(d, context, expressionNode, isEquals(true))
+	return crossFunction(d, context, expressionNode, isEquals(true), false)
 }

pkg/yqlib/operator_multiply.go

@@ -17,7 +17,7 @@ type multiplyPreferences struct {
 
 func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- MultiplyOperator")
-	return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)))
+	return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)), false)
 }
 
 func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

pkg/yqlib/operator_multiply_test.go

@@ -283,5 +283,5 @@ func TestMultiplyOperatorScenarios(t *testing.T) {
 	for _, tt := range multiplyOperatorScenarios {
 		testScenario(t, &tt)
 	}
-	documentScenarios(t, "Multiply", multiplyOperatorScenarios)
+	documentScenarios(t, "Multiply (Merge)", multiplyOperatorScenarios)
 }

pkg/yqlib/operator_strings.go

@@ -3,11 +3,77 @@ package yqlib
 import (
 	"container/list"
 	"fmt"
+	"regexp"
 	"strings"
 
 	"gopkg.in/yaml.v3"
 )
 
+func getSubstituteParameters(d *dataTreeNavigator, block *ExpressionNode, context Context) (string, string, error) {
+	regEx := ""
+	replacementText := ""
+
+	regExNodes, err := d.GetMatchingNodes(context, block.Lhs)
+	if err != nil {
+		return "", "", err
+	}
+	if regExNodes.MatchingNodes.Front() != nil {
+		regEx = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
+	}
+
+	log.Debug("regEx %v", regEx)
+
+	replacementNodes, err := d.GetMatchingNodes(context, block.Rhs)
+	if err != nil {
+		return "", "", err
+	}
+	if replacementNodes.MatchingNodes.Front() != nil {
+		replacementText = replacementNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
+	}
+
+	return regEx, replacementText, nil
+}
+
+func substitute(original string, regex *regexp.Regexp, replacement string) *yaml.Node {
+	replacedString := regex.ReplaceAllString(original, replacement)
+	return &yaml.Node{Kind: yaml.ScalarNode, Value: replacedString, Tag: "!!str"}
+}
+
+func substituteStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	//rhs  block operator
+	//lhs of block = regex
+	//rhs of block = replacement expression
+	block := expressionNode.Rhs
+
+	regExStr, replacementText, err := getSubstituteParameters(d, block, context)
+
+	if err != nil {
+		return Context{}, err
+	}
+
+	regEx, err := regexp.Compile(regExStr)
+	if err != nil {
+		return Context{}, err
+	}
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		node := unwrapDoc(candidate.Node)
+		if node.Tag != "!!str" {
+			return Context{}, fmt.Errorf("cannot sustitute with %v, can only substitute strings", node.Tag)
+		}
+
+		targetNode := substitute(node.Value, regEx, replacementText)
+		result := candidate.CreateChild(nil, targetNode)
+		results.PushBack(result)
+	}
+
+	return context.ChildContext(results), nil
+
+}
+
 func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- joinStringOperator")
 	joinStr := ""
@@ -26,7 +92,7 @@ func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *E
 		candidate := el.Value.(*CandidateNode)
 		node := unwrapDoc(candidate.Node)
 		if node.Kind != yaml.SequenceNode {
-			return Context{}, fmt.Errorf("Cannot join with %v, can only join arrays of scalars", node.Tag)
+			return Context{}, fmt.Errorf("cannot join with %v, can only join arrays of scalars", node.Tag)
 		}
 		targetNode := join(node.Content, joinStr)
 		result := candidate.CreateChild(nil, targetNode)

pkg/yqlib/operator_strings_test.go

@@ -13,6 +13,24 @@ var stringsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!str)::cat; meow; 1; ; true\n",
 		},
 	},
+	{
+		description:    "Substitute / Replace string",
+		subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)",
+		document:       `a: dogs are great`,
+		expression:     `.a |= sub("dogs", "cats")`,
+		expected: []string{
+			"D0, P[], (doc)::a: cats are great\n",
+		},
+	},
+	{
+		description:    "Substitute / Replace string with regex",
+		subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)",
+		document:       "a: cat\nb: heat",
+		expression:     `.[] |= sub("([a])", "${1}r")`,
+		expected: []string{
+			"D0, P[], (doc)::a: cart\nb: heart\n",
+		},
+	},
 	{
 		description: "Split strings",
 		document:    `"cat; meow; 1; ; true"`,

pkg/yqlib/operator_subtract.go

@@ -25,7 +25,7 @@ func subtractAssignOperator(d *dataTreeNavigator, context Context, expressionNod
 func subtractOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("Subtract operator")
 
-	return crossFunction(d, context, expressionNode, subtract)
+	return crossFunction(d, context, expressionNode, subtract, false)
 }
 
 func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

pkg/yqlib/operators.go

@@ -24,7 +24,20 @@ func emptyOperator(d *dataTreeNavigator, context Context, expressionNode *Expres
 
 type crossFunctionCalculation func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)
 
-func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (Context, error) {
+func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *CandidateNode, rhs Context, calculation crossFunctionCalculation, results *list.List) error {
+	for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() {
+		log.Debugf("Applying calc")
+		rhsCandidate := rightEl.Value.(*CandidateNode)
+		resultCandidate, err := calculation(d, context, lhsCandidate, rhsCandidate)
+		if err != nil {
+			return err
+		}
+		results.PushBack(resultCandidate)
+	}
+	return nil
+}
+
+func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation, calcWhenEmpty bool) (Context, error) {
 	var results = list.New()
 	lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
 	if err != nil {
@@ -38,24 +51,33 @@ func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *Expressi
 		return Context{}, err
 	}
 
-	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
-		lhsCandidate := el.Value.(*CandidateNode)
-
-		for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() {
-			log.Debugf("Applying calc")
-			rhsCandidate := rightEl.Value.(*CandidateNode)
-			resultCandidate, err := calculation(d, context, lhsCandidate, rhsCandidate)
+	if calcWhenEmpty && lhs.MatchingNodes.Len() == 0 {
+		if rhs.MatchingNodes.Len() == 0 {
+			resultCandidate, err := calculation(d, context, nil, nil)
 			if err != nil {
 				return Context{}, err
 			}
 			results.PushBack(resultCandidate)
 		}
+		err := resultsForRhs(d, context, nil, rhs, calculation, results)
+		if err != nil {
+			return Context{}, err
+		}
+	}
+
+	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
+		lhsCandidate := el.Value.(*CandidateNode)
+
+		err := resultsForRhs(d, context, lhsCandidate, rhs, calculation, results)
+		if err != nil {
+			return Context{}, err
+		}
 
 	}
 	return context.ChildContext(results), nil
 }
 
-func crossFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (Context, error) {
+func crossFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation, calcWhenEmpty bool) (Context, error) {
 	var results = list.New()
 
 	var evaluateAllTogether = true
@@ -66,11 +88,11 @@ func crossFunction(d *dataTreeNavigator, context Context, expressionNode *Expres
 		}
 	}
 	if evaluateAllTogether {
-		return doCrossFunc(d, context, expressionNode, calculation)
+		return doCrossFunc(d, context, expressionNode, calculation, calcWhenEmpty)
 	}
 
 	for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
-		innerResults, err := doCrossFunc(d, context.SingleChildContext(matchEl.Value.(*CandidateNode)), expressionNode, calculation)
+		innerResults, err := doCrossFunc(d, context.SingleChildContext(matchEl.Value.(*CandidateNode)), expressionNode, calculation, calcWhenEmpty)
 		if err != nil {
 			return Context{}, err
 		}