Added merge if empty

pkg/yqlib/doc/Multiply.md

@@ -111,6 +111,26 @@ b:
   - 5
 ```
 
+## Merge, only existing fields
+Given a sample.yml file of:
+```yaml
+a:
+  thing: one
+  cat: frog
+b:
+  missing: two
+  thing: two
+```
+then
+```bash
+yq eval '.a *? .b' sample.yml
+```
+will output
+```yaml
+thing: two
+cat: frog
+```
+
 ## Merge, appending arrays
 Given a sample.yml file of:
 ```yaml
@@ -143,6 +163,33 @@ array:
 value: banana
 ```
 
+## Merge, only existing fields, appending arrays
+Given a sample.yml file of:
+```yaml
+a:
+  thing:
+    - 1
+    - 2
+b:
+  thing:
+    - 3
+    - 4
+  another:
+    - 1
+```
+then
+```bash
+yq eval '.a *?+ .b' sample.yml
+```
+will output
+```yaml
+thing:
+  - 1
+  - 2
+  - 3
+  - 4
+```
+
 ## Merge to prefix an element
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/expression_processing_test.go

@@ -52,11 +52,6 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "[", "3 (int64)", "]"),
 		append(make([]interface{}, 0), "3 (int64)", "COLLECT", "SHORT_PIPE"),
 	},
-	{
-		`d0.a`,
-		append(make([]interface{}, 0), "d0", "SHORT_PIPE", "a"),
-		append(make([]interface{}, 0), "d0", "a", "SHORT_PIPE"),
-	},
 	{
 		`.a | .[].b == "apple"`,
 		append(make([]interface{}, 0), "a", "PIPE", "TRAVERSE_ARRAY", "[", "]", "SHORT_PIPE", "b", "EQUALS", "apple (string)"),

pkg/yqlib/expression_tokeniser.go

@@ -3,6 +3,7 @@ package yqlib
 import (
 	"fmt"
 	"strconv"
+	"strings"
 
 	lex "github.com/timtadh/lexmachine"
 	"github.com/timtadh/lexmachine/machines"
@@ -65,21 +66,7 @@ func pathToken(wrapped bool) lex.Action {
 			value = unwrap(value)
 		}
 		log.Debug("PathToken %v", value)
-		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value}
+		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: traversePreferences{}}
-		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
-	}
-}
-
-func documentToken() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		var numberString = string(m.Bytes)
-		numberString = numberString[1:]
-		var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) // nolint
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-		log.Debug("documentToken %v", string(m.Bytes))
-		op := &Operation{OperationType: documentFilterOpType, Value: number, StringValue: numberString}
 		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
 	}
 }
@@ -101,6 +88,21 @@ func assignOpToken(updateAssign bool) lex.Action {
 	}
 }
 
+func multiplyWithPrefs() lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		prefs := multiplyPreferences{}
+		options := string(m.Bytes)
+		if strings.Contains(options, "+") {
+			prefs.AppendArrays = true
+		}
+		if strings.Contains(options, "?") {
+			prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
+		}
+		op := &Operation{OperationType: multiplyOpType, Value: multiplyOpType.Type, StringValue: options, Preferences: &prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
 func opTokenWithPrefs(op *operationType, assignOpType *operationType, preferences interface{}) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		log.Debug("opTokenWithPrefs %v", string(m.Bytes))
@@ -220,10 +222,10 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte(`\.\[`), literalToken(traverseArrayCollect, false))
 	lexer.Add([]byte(`\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, &recursiveDescentPreferences{RecurseArray: true,
-		TraversePreferences: &traversePreferences{FollowAlias: false, IncludeMapKeys: false}}))
+		TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: false}}))
 
 	lexer.Add([]byte(`\.\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, &recursiveDescentPreferences{RecurseArray: true,
-		TraversePreferences: &traversePreferences{FollowAlias: false, IncludeMapKeys: true}}))
+		TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}}))
 
 	lexer.Add([]byte(`,`), opToken(unionOpType))
 	lexer.Add([]byte(`:\s*`), opToken(createMapOpType))
@@ -270,7 +272,6 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
 
-	lexer.Add([]byte(`d[0-9]+`), documentToken())
 	lexer.Add([]byte(`\."[^ "]+"`), pathToken(true))
 	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+`), pathToken(false))
 	lexer.Add([]byte(`\.`), selfToken())
@@ -296,7 +297,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
 	lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
 	lexer.Add([]byte(`\*`), opTokenWithPrefs(multiplyOpType, nil, &multiplyPreferences{AppendArrays: false}))
-	lexer.Add([]byte(`\*\+`), opTokenWithPrefs(multiplyOpType, nil, &multiplyPreferences{AppendArrays: true}))
+	lexer.Add([]byte(`\*[\+|\?]*`), multiplyWithPrefs())
 	lexer.Add([]byte(`\+`), opToken(addOpType))
 	lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
 

pkg/yqlib/lib.go

@@ -68,7 +68,6 @@ var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Pre
 var traversePathOpType = &operationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: traversePathOperator}
 var traverseArrayOpType = &operationType{Type: "TRAVERSE_ARRAY", NumArgs: 1, Precedence: 50, Handler: traverseArrayOperator}
 
-var documentFilterOpType = &operationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: traversePathOperator}
 var selfReferenceOpType = &operationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: selfOperator}
 var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: valueOperator}
 var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator}
@@ -120,8 +119,6 @@ func createValueOperation(value interface{}, stringValue string) *Operation {
 func (p *Operation) toString() string {
 	if p.OperationType == traversePathOpType {
 		return fmt.Sprintf("%v", p.Value)
-	} else if p.OperationType == documentFilterOpType {
-		return fmt.Sprintf("d%v", p.Value)
 	} else if p.OperationType == selfReferenceOpType {
 		return "SELF"
 	} else if p.OperationType == valueOpType {

pkg/yqlib/operator_collect_object.go

@@ -60,7 +60,7 @@ func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.
 	candidate := remainingMatches.Remove(remainingMatches.Front()).(*CandidateNode)
 
 	splatted, err := splat(d, nodeToMap(candidate),
-		&traversePreferences{FollowAlias: false, IncludeMapKeys: false})
+		traversePreferences{DontFollowAlias: true, IncludeMapKeys: false})
 
 	for splatEl := splatted.Front(); splatEl != nil; splatEl = splatEl.Next() {
 		splatEl.Value.(*CandidateNode).Path = nil

pkg/yqlib/operator_delete.go

@@ -25,7 +25,7 @@ func deleteChildOperator(d *dataTreeNavigator, matchingNodes *list.List, express
 
 		deleteImmediateChildOpNode := &ExpressionNode{
 			Operation: deleteImmediateChildOp,
-			Rhs:       createTraversalTree(candidate.Path[0 : len(candidate.Path)-1]),
+			Rhs:       createTraversalTree(candidate.Path[0:len(candidate.Path)-1], traversePreferences{}),
 		}
 
 		_, err := d.GetMatchingNodes(matchingNodes, deleteImmediateChildOpNode)

pkg/yqlib/operator_multiply.go

@@ -45,6 +45,7 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, expressionNod
 
 type multiplyPreferences struct {
 	AppendArrays  bool
+	TraversePrefs traversePreferences
 }
 
 func multiplyOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
@@ -59,29 +60,28 @@ func multiply(preferences *multiplyPreferences) func(d *dataTreeNavigator, lhs *
 		log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
 		log.Debugf("-          RHS: %v", rhs.Node.Tag)
 
-		shouldAppendArrays := preferences.AppendArrays
-
 		if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
 			(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
 
 			var newBlank = lhs.CreateChild(nil, &yaml.Node{})
-			var newThing, err = mergeObjects(d, newBlank, lhs, false)
+			var newThing, err = mergeObjects(d, newBlank, lhs, &multiplyPreferences{})
 			if err != nil {
 				return nil, err
 			}
-			return mergeObjects(d, newThing, rhs, shouldAppendArrays)
+			return mergeObjects(d, newThing, rhs, preferences)
 
 		}
 		return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
 	}
 }
 
-func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) (*CandidateNode, error) {
+func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, preferences *multiplyPreferences) (*CandidateNode, error) {
+	shouldAppendArrays := preferences.AppendArrays
 	var results = list.New()
 
 	// shouldn't recurse arrays if appending
 	prefs := &recursiveDescentPreferences{RecurseArray: !shouldAppendArrays,
-		TraversePreferences: &traversePreferences{FollowAlias: false}}
+		TraversePreferences: traversePreferences{DontFollowAlias: true}}
 	err := recursiveDecent(d, results, nodeToMap(rhs), prefs)
 	if err != nil {
 		return nil, err
@@ -93,7 +93,7 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode,
 	}
 
 	for el := results.Front(); el != nil; el = el.Next() {
-		err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), shouldAppendArrays)
+		err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), preferences)
 		if err != nil {
 			return nil, err
 		}
@@ -101,8 +101,8 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode,
 	return lhs, nil
 }
 
-func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) error {
+func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, preferences *multiplyPreferences) error {
-
+	shouldAppendArrays := preferences.AppendArrays
 	log.Debugf("merge - applyAssignment lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs))
 
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
@@ -116,7 +116,7 @@ func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *Candid
 	}
 	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
 
-	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath), Rhs: &ExpressionNode{Operation: rhsOp}}
+	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs), Rhs: &ExpressionNode{Operation: rhsOp}}
 
 	_, err := d.GetMatchingNodes(nodeToMap(lhs), assignmentOpNode)
 

pkg/yqlib/operator_multiply_test.go

@@ -107,6 +107,22 @@ b:
 			"D0, P[a], (!!seq)::[1, 2]\n",
 		},
 	},
+	{
+		description: "Merge, only existing fields",
+		document:    `{a: {thing: one, cat: frog}, b: {missing: two, thing: two}}`,
+		expression:  `.a *? .b`,
+		expected: []string{
+			"D0, P[a], (!!map)::{thing: two, cat: frog}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,
+		expression: `.a *? .b`,
+		expected: []string{
+			"D0, P[a], (!!seq)::[{thing: two}]\n",
+		},
+	},
 	{
 		description: "Merge, appending arrays",
 		document:    `{a: {array: [1, 2, animal: dog], value: coconut}, b: {array: [3, 4, animal: cat], value: banana}}`,
@@ -115,6 +131,14 @@ b:
 			"D0, P[a], (!!map)::{array: [1, 2, {animal: dog}, 3, 4, {animal: cat}], value: banana}\n",
 		},
 	},
+	{
+		description: "Merge, only existing fields, appending arrays",
+		document:    `{a: {thing: [1,2]}, b: {thing: [3,4], another: [1]}}`,
+		expression:  `.a *?+ .b`,
+		expected: []string{
+			"D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n",
+		},
+	},
 	{
 		description: "Merge to prefix an element",
 		document:    `{a: cat, b: dog}`,

pkg/yqlib/operator_recursive_descent.go

@@ -7,7 +7,7 @@ import (
 )
 
 type recursiveDescentPreferences struct {
-	TraversePreferences *traversePreferences
+	TraversePreferences traversePreferences
 	RecurseArray        bool
 }
 

pkg/yqlib/operator_traverse_path.go

@@ -10,11 +10,12 @@ import (
 )
 
 type traversePreferences struct {
-	FollowAlias    bool
+	DontFollowAlias bool
 	IncludeMapKeys  bool
+	DontAutoCreate  bool // by default, we automatically create entries on the fly.
 }
 
-func splat(d *dataTreeNavigator, matches *list.List, prefs *traversePreferences) (*list.List, error) {
+func splat(d *dataTreeNavigator, matches *list.List, prefs traversePreferences) (*list.List, error) {
 	return traverseNodesWithArrayIndices(matches, make([]*yaml.Node, 0), prefs)
 }
 
@@ -54,8 +55,7 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Oper
 	switch value.Kind {
 	case yaml.MappingNode:
 		log.Debug("its a map with %v entries", len(value.Content)/2)
-		prefs := &traversePreferences{FollowAlias: true}
+		return traverseMap(matchingNode, operation.StringValue, operation.Preferences.(traversePreferences), false)
-		return traverseMap(matchingNode, operation.StringValue, prefs, false)
 
 	case yaml.SequenceNode:
 		log.Debug("its a sequence of %v things!", len(value.Content))
@@ -83,11 +83,10 @@ func traverseArrayOperator(d *dataTreeNavigator, matchingNodes *list.List, expre
 	}
 
 	var indicesToTraverse = rhs.Front().Value.(*CandidateNode).Node.Content
-	prefs := &traversePreferences{FollowAlias: true}
+	return traverseNodesWithArrayIndices(matchingNodes, indicesToTraverse, traversePreferences{})
-	return traverseNodesWithArrayIndices(matchingNodes, indicesToTraverse, prefs)
 }
 
-func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, prefs *traversePreferences) (*list.List, error) {
+func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, prefs traversePreferences) (*list.List, error) {
 	var matchingNodeMap = list.New()
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
@@ -101,7 +100,7 @@ func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse [
 	return matchingNodeMap, nil
 }
 
-func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, prefs *traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
+func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, prefs traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
 	node := matchingNode.Node
 	if node.Tag == "!!null" {
 		log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
@@ -124,7 +123,7 @@ func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml
 	return list.New(), nil
 }
 
-func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs *traversePreferences) (*list.List, error) {
+func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs traversePreferences) (*list.List, error) {
 	if len(indices) == 0 {
 		return traverseMap(candidate, "", prefs, true)
 	}
@@ -188,7 +187,7 @@ func keyMatches(key *yaml.Node, wantedKey string) bool {
 	return matchKey(key.Value, wantedKey)
 }
 
-func traverseMap(matchingNode *CandidateNode, key string, prefs *traversePreferences, splat bool) (*list.List, error) {
+func traverseMap(matchingNode *CandidateNode, key string, prefs traversePreferences, splat bool) (*list.List, error) {
 	var newMatches = orderedmap.NewOrderedMap()
 	err := doTraverseMap(newMatches, matchingNode, key, prefs, splat)
 
@@ -196,7 +195,7 @@ func traverseMap(matchingNode *CandidateNode, key string, prefs *traversePrefere
 		return nil, err
 	}
 
-	if newMatches.Len() == 0 {
+	if !prefs.DontAutoCreate && newMatches.Len() == 0 {
 		//no matches, create one automagically
 		valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}
 		node := matchingNode.Node
@@ -214,7 +213,7 @@ func traverseMap(matchingNode *CandidateNode, key string, prefs *traversePrefere
 	return results, nil
 }
 
-func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, prefs *traversePreferences, splat bool) error {
+func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, prefs traversePreferences, splat bool) error {
 	// value.Content is a concatenated array of key, value,
 	// so keys are in the even indexes, values in odd.
 	// merge aliases are defined first, but we only want to traverse them
@@ -229,7 +228,7 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode,
 
 		log.Debug("checking %v (%v)", key.Value, key.Tag)
 		//skip the 'merge' tag, find a direct match first
-		if key.Tag == "!!merge" && prefs.FollowAlias {
+		if key.Tag == "!!merge" && !prefs.DontFollowAlias {
 			log.Debug("Merge anchor")
 			err := traverseMergeAnchor(newMatches, candidate, value, wantedKey, prefs, splat)
 			if err != nil {
@@ -249,7 +248,7 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode,
 	return nil
 }
 
-func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, prefs *traversePreferences, splat bool) error {
+func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, prefs traversePreferences, splat bool) error {
 	switch value.Kind {
 	case yaml.AliasNode:
 		candidateNode := originalCandidate.CreateChild(nil, value.Alias)

pkg/yqlib/operators.go

@@ -35,14 +35,16 @@ func nodeToMap(candidate *CandidateNode) *list.List {
 	return elMap
 }
 
-func createTraversalTree(path []interface{}) *ExpressionNode {
+func createTraversalTree(path []interface{}, traversePrefs traversePreferences) *ExpressionNode {
 	if len(path) == 0 {
 		return &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
 	} else if len(path) == 1 {
-		return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
+		return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Preferences: traversePrefs, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
 	}
+
 	return &ExpressionNode{
 		Operation: &Operation{OperationType: shortPipeOpType},
-		Lhs:       createTraversalTree(path[0:1]),
+		Lhs:       createTraversalTree(path[0:1], traversePrefs),
-		Rhs:       createTraversalTree(path[1:])}
+		Rhs:       createTraversalTree(path[1:], traversePrefs),
+	}
 }