attempt to fix dockre

json.Encoder and json.Marshal implicitly use HTMLEscape to convert
>, <, &, with \u003c, \u003e, \u0026. This behavior carries over
to yq, where chars will be escaped when outputting json but not when
outputting yaml.

This changeset disables this behavior via encoder.SetEscapeHTML(false).
Unfortunately there is no equivalent option for json.Marshal, so its
single usage has been replaced with an encoder (with escaping disabled).

.github/workflows/release.yml

@@ -3,6 +3,7 @@ on:
   push:
     tags:
       - 'v*'
+      - 'dockerfix'
 
 jobs:
   publishGitRelease:

README.md

@@ -90,7 +90,7 @@ docker run --rm -v "${PWD}":/workdir mikefarah/yq <command> [flags] [expression
 #### Run commands interactively:
 
 ```bash
-docker run --rm -it -v "${PWD}":/workdir mikefarah/yq sh
+docker run --rm -it -v "${PWD}":/workdir --entrypoint sh mikefarah/yq
 ```
 
 It can be useful to have a bash function to avoid typing the whole docker command:

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.3.2"
+	Version = "4.4.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.3.2
+FROM mikefarah/yq:4.4.1
 
 COPY entrypoint.sh /entrypoint.sh
 

pkg/yqlib/candidate_node.go

@@ -15,6 +15,9 @@ type CandidateNode struct {
 	Document  uint          // the document index of this node
 	Filename  string
 	FileIndex int
+	// when performing op against all nodes given, this will treat all the nodes as one
+	// (e.g. top level cross document merge). This property does not propegate to child nodes.
+	EvaluateTogether bool
 }
 
 func (n *CandidateNode) GetKey() string {

pkg/yqlib/doc/Add.md

@@ -209,3 +209,15 @@ will output
 a: 4
 ```
 
+## Add to null
+Adding to null simply returns the rhs
+
+Running
+```bash
+yq eval --null-input 'null + "cat"'
+```
+will output
+```yaml
+cat
+```
+

pkg/yqlib/doc/Comment Operators.md

@@ -78,17 +78,22 @@ b: dog # leave this
 ```
 
 ## Remove all comments
+Note the use of `...` to ensure key nodes are included.
+
 Given a sample.yml file of:
 ```yaml
 a: cat # comment
+# great
+b: # key comment
 ```
 then
 ```bash
-yq eval '.. comments=""' sample.yml
+yq eval '... comments=""' sample.yml
 ```
 will output
 ```yaml
 a: cat
+b:
 ```
 
 ## Get line comment

pkg/yqlib/doc/Keys.md

@@ -0,0 +1,35 @@
+# Keys
+
+Use the `keys` operator to return map keys or array indices. 
+## Map keys
+Given a sample.yml file of:
+```yaml
+dog: woof
+cat: meow
+```
+then
+```bash
+yq eval 'keys' sample.yml
+```
+will output
+```yaml
+- dog
+- cat
+```
+
+## Array keys
+Given a sample.yml file of:
+```yaml
+- apple
+- banana
+```
+then
+```bash
+yq eval 'keys' sample.yml
+```
+will output
+```yaml
+- 0
+- 1
+```
+

pkg/yqlib/doc/Multiply.md

@@ -1,13 +1,14 @@
-Like the multiple operator in `jq`, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
+Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
 
-Upcoming versions of `yq` will add support for other types of multiplication (numbers, strings).
-
-To concatenate when merging objects, use the `*+` form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
+To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
 
+To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?.
 Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
 
+Multiplication of strings and numbers are not yet supported.
+
 ## Merging files
-Note the use of eval-all to ensure all documents are loaded into memory.
+Note the use of `eval-all` to ensure all documents are loaded into memory.
 
 ```bash
 yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml
@@ -111,6 +112,26 @@ b:
   - 5
 ```
 
+## Merge, only existing fields
+Given a sample.yml file of:
+```yaml
+a:
+  thing: one
+  cat: frog
+b:
+  missing: two
+  thing: two
+```
+then
+```bash
+yq eval '.a *? .b' sample.yml
+```
+will output
+```yaml
+thing: two
+cat: frog
+```
+
 ## Merge, appending arrays
 Given a sample.yml file of:
 ```yaml
@@ -143,6 +164,33 @@ array:
 value: banana
 ```
 
+## Merge, only existing fields, appending arrays
+Given a sample.yml file of:
+```yaml
+a:
+  thing:
+    - 1
+    - 2
+b:
+  thing:
+    - 3
+    - 4
+  another:
+    - 1
+```
+then
+```bash
+yq eval '.a *?+ .b' sample.yml
+```
+will output
+```yaml
+thing:
+  - 1
+  - 2
+  - 3
+  - 4
+```
+
 ## Merge to prefix an element
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Split into Documents.md

@@ -0,0 +1,31 @@
+# Split into Documents
+
+This operator splits all matches into separate documents
+
+## Split empty
+Running
+```bash
+yq eval --null-input 'splitDoc'
+```
+will output
+```yaml
+
+```
+
+## Split array
+Given a sample.yml file of:
+```yaml
+- a: cat
+- b: dog
+```
+then
+```bash
+yq eval '.[] | splitDoc' sample.yml
+```
+will output
+```yaml
+a: cat
+---
+b: dog
+```
+

pkg/yqlib/doc/String Operators.md

@@ -0,0 +1,52 @@
+# String Operators
+
+## Join strings
+Given a sample.yml file of:
+```yaml
+- cat
+- meow
+- 1
+- null
+- true
+```
+then
+```bash
+yq eval 'join("; ")' sample.yml
+```
+will output
+```yaml
+cat; meow; 1; ; true
+```
+
+## Split strings
+Given a sample.yml file of:
+```yaml
+cat; meow; 1; ; true
+```
+then
+```bash
+yq eval 'split("; ")' sample.yml
+```
+will output
+```yaml
+- cat
+- meow
+- "1"
+- ""
+- "true"
+```
+
+## Split strings one match
+Given a sample.yml file of:
+```yaml
+word
+```
+then
+```bash
+yq eval 'split("; ")' sample.yml
+```
+will output
+```yaml
+- word
+```
+

pkg/yqlib/doc/aa.md

@@ -0,0 +1,12 @@
+# Operators
+
+In `yq` expressions are made up of operators. Operators have 0-2 arguments and run against the current 'matching' nodes in the expression tree.
+
+Lets look at a couple of examples.
+
+The `length` operator take no arguments, and will simply return the length of _each_ matching node. So if there were 2 nodes, one string and one array, length will update the 'matching' nodes context to be two new numeric scalar nodes representing the lengths of the orignal 'matching' nodes.
+
+The `=` operator takes two arguments, a `lhs` expression and `rhs` expression. It runs the 'matching' nodes context against the `lhs` expression to find the nodes to update, lets call it `lhsNodes`, and then runs the matching nodes against the `rhs` to find the new values, lets call that `rhsNodes`. It updates the `lhsNodes` values with the `rhsNodes` values and _returns the original matching nodes_. This is important, where length changed the matching nodes to be new nodes with the length values, `=` returns the original matching nodes, albeit with some of the nodes values updated. So `.a = 3` will still return the parent matching node, but with the matching child updated.
+
+Please see the individual operator docs for more information and examples.
+

pkg/yqlib/doc/headers/Keys.md

@@ -0,0 +1,3 @@
+# Keys
+
+Use the `keys` operator to return map keys or array indices. 

pkg/yqlib/doc/headers/Multiply.md

@@ -1,13 +1,14 @@
-Like the multiple operator in `jq`, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
+Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
 
-Upcoming versions of `yq` will add support for other types of multiplication (numbers, strings).
-
-To concatenate when merging objects, use the `*+` form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
+To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
 
+To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?.
 Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
 
+Multiplication of strings and numbers are not yet supported.
+
 ## Merging files
-Note the use of eval-all to ensure all documents are loaded into memory.
+Note the use of `eval-all` to ensure all documents are loaded into memory.
 
 ```bash
 yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml

pkg/yqlib/doc/headers/Split into Documents.md

@@ -0,0 +1,3 @@
+# Split into Documents
+
+This operator splits all matches into separate documents

pkg/yqlib/doc/headers/String Operators.md

@@ -0,0 +1 @@
+# String Operators

pkg/yqlib/encoder.go

@@ -76,6 +76,8 @@ func mapKeysToStrings(node *yaml.Node) {
 
 func NewJsonEncoder(destination io.Writer, indent int) Encoder {
 	var encoder = json.NewEncoder(destination)
+	encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
+
 	var indentString = ""
 
 	for index := 0; index < indent; index++ {
@@ -153,11 +155,15 @@ func (o *orderedMap) UnmarshalJSON(data []byte) error {
 }
 
 func (o orderedMap) MarshalJSON() ([]byte, error) {
-	if o.kv == nil {
-		return json.Marshal(o.altVal)
-	}
 	buf := new(bytes.Buffer)
 	enc := json.NewEncoder(buf)
+	enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
+	if o.kv == nil {
+		if err := enc.Encode(o.altVal); err != nil {
+			return nil, err
+		}
+		return buf.Bytes(), nil
+	}
 	buf.WriteByte('{')
 	for idx, el := range o.kv {
 		if err := enc.Encode(el.K); err != nil {

pkg/yqlib/encoder_test.go

@@ -9,29 +9,11 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-var sampleYaml = `zabbix: winner
-apple: great
-banana:
-- {cobra: kai, angus: bob}
-`
-
-var expectedJson = `{
-  "zabbix": "winner",
-  "apple": "great",
-  "banana": [
-    {
-      "cobra": "kai",
-      "angus": "bob"
-    }
-  ]
-}
-`
-
-func TestJsonEncoderPreservesObjectOrder(t *testing.T) {
+func yamlToJson(sampleYaml string, indent int) string {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
-	var jsonEncoder = NewJsonEncoder(writer, 2)
+	var jsonEncoder = NewJsonEncoder(writer, indent)
 	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0)
 	if err != nil {
 		panic(err)
@@ -42,6 +24,33 @@ func TestJsonEncoderPreservesObjectOrder(t *testing.T) {
 		panic(err)
 	}
 	writer.Flush()
-	test.AssertResult(t, expectedJson, output.String())
 
+	return strings.TrimSuffix(output.String(), "\n")
+}
+
+func TestJsonEncoderPreservesObjectOrder(t *testing.T) {
+	var sampleYaml = `zabbix: winner
+apple: great
+banana:
+- {cobra: kai, angus: bob}
+`
+	var expectedJson = `{
+  "zabbix": "winner",
+  "apple": "great",
+  "banana": [
+    {
+      "cobra": "kai",
+      "angus": "bob"
+    }
+  ]
+}`
+	var actualJson = yamlToJson(sampleYaml, 2)
+	test.AssertResult(t, expectedJson, actualJson)
+}
+
+func TestJsonEncoderDoesNotEscapeHTMLChars(t *testing.T) {
+	var sampleYaml = `build: "( ./lint && ./format && ./compile ) < src.code"`
+	var expectedJson = `{"build":"( ./lint && ./format && ./compile ) < src.code"}`
+	var actualJson = yamlToJson(sampleYaml, 0)
+	test.AssertResult(t, expectedJson, actualJson)
 }

pkg/yqlib/expression_processing_test.go

@@ -53,9 +53,19 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "3 (int64)", "COLLECT", "SHORT_PIPE"),
 	},
 	{
-		`d0.a`,
-		append(make([]interface{}, 0), "d0", "SHORT_PIPE", "a"),
-		append(make([]interface{}, 0), "d0", "a", "SHORT_PIPE"),
+		`.key.array + .key.array2`,
+		append(make([]interface{}, 0), "key", "SHORT_PIPE", "array", "ADD", "key", "SHORT_PIPE", "array2"),
+		append(make([]interface{}, 0), "key", "array", "SHORT_PIPE", "key", "array2", "SHORT_PIPE", "ADD"),
+	},
+	{
+		`.key.array * .key.array2`,
+		append(make([]interface{}, 0), "key", "SHORT_PIPE", "array", "MULTIPLY", "key", "SHORT_PIPE", "array2"),
+		append(make([]interface{}, 0), "key", "array", "SHORT_PIPE", "key", "array2", "SHORT_PIPE", "MULTIPLY"),
+	},
+	{
+		`.key.array // .key.array2`,
+		append(make([]interface{}, 0), "key", "SHORT_PIPE", "array", "ALTERNATIVE", "key", "SHORT_PIPE", "array2"),
+		append(make([]interface{}, 0), "key", "array", "SHORT_PIPE", "key", "array2", "SHORT_PIPE", "ALTERNATIVE"),
 	},
 	{
 		`.a | .[].b == "apple"`,

pkg/yqlib/expression_tokeniser.go

@@ -3,6 +3,7 @@ package yqlib
 import (
 	"fmt"
 	"strconv"
+	"strings"
 
 	lex "github.com/timtadh/lexmachine"
 	"github.com/timtadh/lexmachine/machines"
@@ -65,21 +66,7 @@ func pathToken(wrapped bool) lex.Action {
 			value = unwrap(value)
 		}
 		log.Debug("PathToken %v", value)
-		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value}
-		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
-	}
-}
-
-func documentToken() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		var numberString = string(m.Bytes)
-		numberString = numberString[1:]
-		var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) // nolint
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-		log.Debug("documentToken %v", string(m.Bytes))
-		op := &Operation{OperationType: documentFilterOpType, Value: number, StringValue: numberString}
+		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: traversePreferences{}}
 		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
 	}
 }
@@ -101,6 +88,21 @@ func assignOpToken(updateAssign bool) lex.Action {
 	}
 }
 
+func multiplyWithPrefs() lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		prefs := multiplyPreferences{}
+		options := string(m.Bytes)
+		if strings.Contains(options, "+") {
+			prefs.AppendArrays = true
+		}
+		if strings.Contains(options, "?") {
+			prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
+		}
+		op := &Operation{OperationType: multiplyOpType, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
 func opTokenWithPrefs(op *operationType, assignOpType *operationType, preferences interface{}) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		log.Debug("opTokenWithPrefs %v", string(m.Bytes))
@@ -123,7 +125,7 @@ func assignAllCommentsOp(updateAssign bool) lex.Action {
 			Value:         assignCommentOpType.Type,
 			StringValue:   value,
 			UpdateAssign:  updateAssign,
-			Preferences:   &commentOpPreferences{LineComment: true, HeadComment: true, FootComment: true},
+			Preferences:   commentOpPreferences{LineComment: true, HeadComment: true, FootComment: true},
 		}
 		return &token{TokenType: operationToken, Operation: op}, nil
 	}
@@ -181,7 +183,7 @@ func stringValue(wrapped bool) lex.Action {
 func envOp(strenv bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		value := string(m.Bytes)
-		preferences := &envOpPreferences{}
+		preferences := envOpPreferences{}
 
 		if strenv {
 			// strenv( )
@@ -219,11 +221,11 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`\)`), literalToken(closeBracket, true))
 
 	lexer.Add([]byte(`\.\[`), literalToken(traverseArrayCollect, false))
-	lexer.Add([]byte(`\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, &recursiveDescentPreferences{RecurseArray: true,
-		TraversePreferences: &traversePreferences{FollowAlias: false, IncludeMapKeys: false}}))
+	lexer.Add([]byte(`\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, recursiveDescentPreferences{RecurseArray: true,
+		TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: false}}))
 
-	lexer.Add([]byte(`\.\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, &recursiveDescentPreferences{RecurseArray: true,
-		TraversePreferences: &traversePreferences{FollowAlias: false, IncludeMapKeys: true}}))
+	lexer.Add([]byte(`\.\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, recursiveDescentPreferences{RecurseArray: true,
+		TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}}))
 
 	lexer.Add([]byte(`,`), opToken(unionOpType))
 	lexer.Add([]byte(`:\s*`), opToken(createMapOpType))
@@ -239,6 +241,11 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte(`documentIndex`), opToken(getDocumentIndexOpType))
 	lexer.Add([]byte(`di`), opToken(getDocumentIndexOpType))
+	lexer.Add([]byte(`splitDoc`), opToken(splitDocumentOpType))
+
+	lexer.Add([]byte(`join`), opToken(joinStringOpType))
+	lexer.Add([]byte(`split`), opToken(splitStringOpType))
+	lexer.Add([]byte(`keys`), opToken(keysOpType))
 
 	lexer.Add([]byte(`style`), opAssignableToken(getStyleOpType, assignStyleOpType))
 
@@ -250,11 +257,11 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`fi`), opToken(getFileIndexOpType))
 	lexer.Add([]byte(`path`), opToken(getPathOpType))
 
-	lexer.Add([]byte(`lineComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, &commentOpPreferences{LineComment: true}))
+	lexer.Add([]byte(`lineComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
 
-	lexer.Add([]byte(`headComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, &commentOpPreferences{HeadComment: true}))
+	lexer.Add([]byte(`headComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}))
 
-	lexer.Add([]byte(`footComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, &commentOpPreferences{FootComment: true}))
+	lexer.Add([]byte(`footComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}))
 
 	lexer.Add([]byte(`comments\s*=`), assignAllCommentsOp(false))
 	lexer.Add([]byte(`comments\s*\|=`), assignAllCommentsOp(true))
@@ -270,7 +277,6 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
 
-	lexer.Add([]byte(`d[0-9]+`), documentToken())
 	lexer.Add([]byte(`\."[^ "]+"`), pathToken(true))
 	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+`), pathToken(false))
 	lexer.Add([]byte(`\.`), selfToken())
@@ -295,8 +301,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`\]`), literalToken(closeCollect, true))
 	lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
 	lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
-	lexer.Add([]byte(`\*`), opTokenWithPrefs(multiplyOpType, nil, &multiplyPreferences{AppendArrays: false}))
-	lexer.Add([]byte(`\*\+`), opTokenWithPrefs(multiplyOpType, nil, &multiplyPreferences{AppendArrays: true}))
+	lexer.Add([]byte(`\*[\+|\?]*`), multiplyWithPrefs())
 	lexer.Add([]byte(`\+`), opToken(addOpType))
 	lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
 

pkg/yqlib/lib.go

@@ -40,9 +40,9 @@ var assignCommentOpType = &operationType{Type: "ASSIGN_COMMENT", NumArgs: 2, Pre
 var assignAnchorOpType = &operationType{Type: "ASSIGN_ANCHOR", NumArgs: 2, Precedence: 40, Handler: assignAnchorOperator}
 var assignAliasOpType = &operationType{Type: "ASSIGN_ALIAS", NumArgs: 2, Precedence: 40, Handler: assignAliasOperator}
 
-var multiplyOpType = &operationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 45, Handler: multiplyOperator}
-var addOpType = &operationType{Type: "ADD", NumArgs: 2, Precedence: 45, Handler: addOperator}
-var alternativeOpType = &operationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 45, Handler: alternativeOperator}
+var multiplyOpType = &operationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 42, Handler: multiplyOperator}
+var addOpType = &operationType{Type: "ADD", NumArgs: 2, Precedence: 42, Handler: addOperator}
+var alternativeOpType = &operationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 42, Handler: alternativeOperator}
 
 var equalsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: equalsOperator}
 var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 40, Handler: createMapOperator}
@@ -51,6 +51,7 @@ var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence:
 
 var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: collectOperator}
+var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator}
 var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}
 var getTagOpType = &operationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: getTagOperator}
 var getCommentOpType = &operationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: getCommentsOperator}
@@ -63,12 +64,15 @@ var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50,
 
 var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator}
 var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
+var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}
+var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
+
+var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}
 
 var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}
 var traversePathOpType = &operationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: traversePathOperator}
 var traverseArrayOpType = &operationType{Type: "TRAVERSE_ARRAY", NumArgs: 1, Precedence: 50, Handler: traverseArrayOperator}
 
-var documentFilterOpType = &operationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: traversePathOperator}
 var selfReferenceOpType = &operationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: selfOperator}
 var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: valueOperator}
 var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator}
@@ -120,8 +124,6 @@ func createValueOperation(value interface{}, stringValue string) *Operation {
 func (p *Operation) toString() string {
 	if p.OperationType == traversePathOpType {
 		return fmt.Sprintf("%v", p.Value)
-	} else if p.OperationType == documentFilterOpType {
-		return fmt.Sprintf("d%v", p.Value)
 	} else if p.OperationType == selfReferenceOpType {
 		return "SELF"
 	} else if p.OperationType == valueOpType {

pkg/yqlib/operator_add.go

@@ -47,9 +47,14 @@ func add(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*Candida
 	lhs.Node = unwrapDoc(lhs.Node)
 	rhs.Node = unwrapDoc(rhs.Node)
 
-	target := lhs.CreateChild(nil, &yaml.Node{})
 	lhsNode := lhs.Node
 
+	if lhsNode.Tag == "!!null" {
+		return lhs.CreateChild(nil, rhs.Node), nil
+	}
+
+	target := lhs.CreateChild(nil, &yaml.Node{})
+
 	switch lhsNode.Kind {
 	case yaml.MappingNode:
 		return nil, fmt.Errorf("Maps not yet supported for addition")

pkg/yqlib/operator_add_test.go

@@ -5,6 +5,15 @@ import (
 )
 
 var addOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `[{a: foo, b: bar}, {a: 1, b: 2}]`,
+		expression: ".[] | .a + .b",
+		expected: []string{
+			"D0, P[0 a], (!!str)::foobar\n",
+			"D0, P[1 a], (!!int)::3\n",
+		},
+	},
 	{
 		description: "Concatenate and assign arrays",
 		document:    `{a: {val: thing, b: [cat,dog]}}`,
@@ -103,6 +112,14 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: 4}\n",
 		},
 	},
+	{
+		description:    "Add to null",
+		subdescription: "Adding to null simply returns the rhs",
+		expression:     `null + "cat"`,
+		expected: []string{
+			"D0, P[], (!!str)::cat\n",
+		},
+	},
 }
 
 func TestAddOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_collect_object.go

@@ -60,7 +60,7 @@ func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.
 	candidate := remainingMatches.Remove(remainingMatches.Front()).(*CandidateNode)
 
 	splatted, err := splat(d, nodeToMap(candidate),
-		&traversePreferences{FollowAlias: false, IncludeMapKeys: false})
+		traversePreferences{DontFollowAlias: true, IncludeMapKeys: false})
 
 	for splatEl := splatted.Front(); splatEl != nil; splatEl = splatEl.Next() {
 		splatEl.Value.(*CandidateNode).Path = nil
@@ -87,7 +87,7 @@ func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.
 
 			newCandidate.Path = nil
 
-			newCandidate, err = multiply(&multiplyPreferences{AppendArrays: false})(d, newCandidate, splatCandidate)
+			newCandidate, err = multiply(multiplyPreferences{AppendArrays: false})(d, newCandidate, splatCandidate)
 			if err != nil {
 				return nil, err
 			}

pkg/yqlib/operator_comments.go

@@ -23,7 +23,7 @@ func assignCommentsOperator(d *dataTreeNavigator, matchingNodes *list.List, expr
 		return nil, err
 	}
 
-	preferences := expressionNode.Operation.Preferences.(*commentOpPreferences)
+	preferences := expressionNode.Operation.Preferences.(commentOpPreferences)
 
 	comment := ""
 	if !expressionNode.Operation.UpdateAssign {
@@ -67,7 +67,7 @@ func assignCommentsOperator(d *dataTreeNavigator, matchingNodes *list.List, expr
 }
 
 func getCommentsOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
-	preferences := expressionNode.Operation.Preferences.(*commentOpPreferences)
+	preferences := expressionNode.Operation.Preferences.(commentOpPreferences)
 	log.Debugf("GetComments operator!")
 	var results = list.New()
 

pkg/yqlib/operator_comments_test.go

@@ -71,11 +71,12 @@ var commentOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Remove all comments",
-		document:    "# hi\n\na: cat # comment\n\n# great\n",
-		expression:  `.. comments=""`,
+		description:    "Remove all comments",
+		subdescription: "Note the use of `...` to ensure key nodes are included.",
+		document:       "# hi\n\na: cat # comment\n\n# great\n\nb: # key comment",
+		expression:     `... comments=""`,
 		expected: []string{
-			"D0, P[], (!!map)::a: cat\n",
+			"D0, P[], (!!map)::a: cat\nb:\n",
 		},
 	},
 	{

pkg/yqlib/operator_delete.go

@@ -25,7 +25,7 @@ func deleteChildOperator(d *dataTreeNavigator, matchingNodes *list.List, express
 
 		deleteImmediateChildOpNode := &ExpressionNode{
 			Operation: deleteImmediateChildOp,
-			Rhs:       createTraversalTree(candidate.Path[0 : len(candidate.Path)-1]),
+			Rhs:       createTraversalTree(candidate.Path[0:len(candidate.Path)-1], traversePreferences{}),
 		}
 
 		_, err := d.GetMatchingNodes(matchingNodes, deleteImmediateChildOpNode)

pkg/yqlib/operator_env.go

@@ -19,7 +19,7 @@ func envOperator(d *dataTreeNavigator, matchMap *list.List, expressionNode *Expr
 
 	rawValue := os.Getenv(envName)
 
-	preferences := expressionNode.Operation.Preferences.(*envOpPreferences)
+	preferences := expressionNode.Operation.Preferences.(envOpPreferences)
 
 	var node *yaml.Node
 	if preferences.StringValue {

pkg/yqlib/operator_keys.go

@@ -0,0 +1,54 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+
+	"gopkg.in/yaml.v3"
+)
+
+func keysOperator(d *dataTreeNavigator, matchMap *list.List, expressionNode *ExpressionNode) (*list.List, error) {
+	log.Debugf("-- keysOperator")
+
+	var results = list.New()
+
+	for el := matchMap.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		node := unwrapDoc(candidate.Node)
+		var targetNode *yaml.Node
+		if node.Kind == yaml.MappingNode {
+			targetNode = getMapKeys(node)
+		} else if node.Kind == yaml.SequenceNode {
+			targetNode = getIndicies(node)
+		} else {
+			return nil, fmt.Errorf("Cannot get keys of %v, keys only works for maps and arrays", node.Tag)
+		}
+
+		result := candidate.CreateChild(nil, targetNode)
+		results.PushBack(result)
+	}
+
+	return results, nil
+}
+
+func getMapKeys(node *yaml.Node) *yaml.Node {
+	contents := make([]*yaml.Node, 0)
+	for index := 0; index < len(node.Content); index = index + 2 {
+		contents = append(contents, node.Content[index])
+	}
+	return &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq", Content: contents}
+}
+
+func getIndicies(node *yaml.Node) *yaml.Node {
+	var contents = make([]*yaml.Node, len(node.Content))
+
+	for index := range node.Content {
+		contents[index] = &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!int",
+			Value: fmt.Sprintf("%v", index),
+		}
+	}
+
+	return &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq", Content: contents}
+}

pkg/yqlib/operator_keys_test.go

@@ -0,0 +1,47 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var keysOperatorScenarios = []expressionScenario{
+	{
+		description: "Map keys",
+		document:    `{dog: woof, cat: meow}`,
+		expression:  `keys`,
+		expected: []string{
+			"D0, P[], (!!seq)::- dog\n- cat\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: `keys`,
+		expected: []string{
+			"D0, P[], (!!seq)::[]\n",
+		},
+	},
+	{
+		description: "Array keys",
+		document:    `[apple, banana]`,
+		expression:  `keys`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 0\n- 1\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[]`,
+		expression: `keys`,
+		expected: []string{
+			"D0, P[], (!!seq)::[]\n",
+		},
+	},
+}
+
+func TestKeysOperatorScenarios(t *testing.T) {
+	for _, tt := range keysOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Keys", keysOperatorScenarios)
+}

pkg/yqlib/operator_multiply.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"fmt"
+	"strconv"
 
 	"container/list"
 
@@ -10,21 +11,19 @@ import (
 
 type crossFunctionCalculation func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)
 
-func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (*list.List, error) {
-	lhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Lhs)
+func doCrossFunc(d *dataTreeNavigator, contextList *list.List, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (*list.List, error) {
+	var results = list.New()
+	lhs, err := d.GetMatchingNodes(contextList, expressionNode.Lhs)
 	if err != nil {
 		return nil, err
 	}
 	log.Debugf("crossFunction LHS len: %v", lhs.Len())
 
-	rhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Rhs)
+	rhs, err := d.GetMatchingNodes(contextList, expressionNode.Rhs)
 
 	if err != nil {
 		return nil, err
 	}
-	log.Debugf("crossFunction RHS len: %v", rhs.Len())
-
-	var results = list.New()
 
 	for el := lhs.Front(); el != nil; el = el.Next() {
 		lhsCandidate := el.Value.(*CandidateNode)
@@ -43,45 +42,90 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, expressionNod
 	return results, nil
 }
 
+func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (*list.List, error) {
+	var results = list.New()
+
+	var evaluateAllTogether = true
+	for matchEl := matchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
+		evaluateAllTogether = evaluateAllTogether && matchEl.Value.(*CandidateNode).EvaluateTogether
+		if !evaluateAllTogether {
+			break
+		}
+	}
+	if evaluateAllTogether {
+		return doCrossFunc(d, matchingNodes, expressionNode, calculation)
+	}
+
+	for matchEl := matchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
+		contextList := nodeToMap(matchEl.Value.(*CandidateNode))
+		innerResults, err := doCrossFunc(d, contextList, expressionNode, calculation)
+		if err != nil {
+			return nil, err
+		}
+		results.PushBackList(innerResults)
+	}
+
+	return results, nil
+}
+
 type multiplyPreferences struct {
-	AppendArrays bool
+	AppendArrays  bool
+	TraversePrefs traversePreferences
 }
 
 func multiplyOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	log.Debugf("-- MultiplyOperator")
-	return crossFunction(d, matchingNodes, expressionNode, multiply(expressionNode.Operation.Preferences.(*multiplyPreferences)))
+	return crossFunction(d, matchingNodes, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)))
 }
 
-func multiply(preferences *multiplyPreferences) func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		lhs.Node = unwrapDoc(lhs.Node)
 		rhs.Node = unwrapDoc(rhs.Node)
 		log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
 		log.Debugf("-          RHS: %v", rhs.Node.Tag)
 
-		shouldAppendArrays := preferences.AppendArrays
-
 		if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
 			(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
 
 			var newBlank = lhs.CreateChild(nil, &yaml.Node{})
-			var newThing, err = mergeObjects(d, newBlank, lhs, false)
+			var newThing, err = mergeObjects(d, newBlank, lhs, multiplyPreferences{})
 			if err != nil {
 				return nil, err
 			}
-			return mergeObjects(d, newThing, rhs, shouldAppendArrays)
-
+			return mergeObjects(d, newThing, rhs, preferences)
+		} else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" {
+			return multiplyIntegers(lhs, rhs)
 		}
 		return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
 	}
 }
 
-func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) (*CandidateNode, error) {
+func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	target := lhs.CreateChild(nil, &yaml.Node{})
+	target.Node.Kind = yaml.ScalarNode
+	target.Node.Style = lhs.Node.Style
+	target.Node.Tag = "!!int"
+
+	lhsNum, err := strconv.Atoi(lhs.Node.Value)
+	if err != nil {
+		return nil, err
+	}
+	rhsNum, err := strconv.Atoi(rhs.Node.Value)
+	if err != nil {
+		return nil, err
+	}
+	target.Node.Value = fmt.Sprintf("%v", lhsNum*rhsNum)
+	return target, nil
+}
+
+func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) (*CandidateNode, error) {
+	shouldAppendArrays := preferences.AppendArrays
 	var results = list.New()
 
 	// shouldn't recurse arrays if appending
-	prefs := &recursiveDescentPreferences{RecurseArray: !shouldAppendArrays,
-		TraversePreferences: &traversePreferences{FollowAlias: false}}
+	prefs := recursiveDescentPreferences{RecurseArray: !shouldAppendArrays,
+		TraversePreferences: traversePreferences{DontFollowAlias: true}}
 	err := recursiveDecent(d, results, nodeToMap(rhs), prefs)
 	if err != nil {
 		return nil, err
@@ -93,7 +137,7 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode,
 	}
 
 	for el := results.Front(); el != nil; el = el.Next() {
-		err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), shouldAppendArrays)
+		err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), preferences)
 		if err != nil {
 			return nil, err
 		}
@@ -101,8 +145,8 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode,
 	return lhs, nil
 }
 
-func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) error {
-
+func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) error {
+	shouldAppendArrays := preferences.AppendArrays
 	log.Debugf("merge - applyAssignment lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs))
 
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
@@ -116,7 +160,7 @@ func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *Candid
 	}
 	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
 
-	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath), Rhs: &ExpressionNode{Operation: rhsOp}}
+	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs), Rhs: &ExpressionNode{Operation: rhsOp}}
 
 	_, err := d.GetMatchingNodes(nodeToMap(lhs), assignmentOpNode)
 

pkg/yqlib/operator_multiply_test.go

@@ -13,6 +13,27 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{a: 2, b: 5}`,
+		document2:  `{a: 3, b: 10}`,
+		expression: `.a * .b`,
+		expected: []string{
+			"D0, P[a], (!!int)::10\n",
+			"D0, P[a], (!!int)::20\n",
+			"D0, P[a], (!!int)::15\n",
+			"D0, P[a], (!!int)::30\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: 2}`,
+		document2:  `{b: 10}`,
+		expression: `select(fi ==0) * select(fi==1)`,
+		expected: []string{
+			"D0, P[], (!!map)::{a: 2, b: 10}\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		expression: `{} * {"cat":"dog"}`,
@@ -107,6 +128,30 @@ b:
 			"D0, P[a], (!!seq)::[1, 2]\n",
 		},
 	},
+	{
+		description: "Merge, only existing fields",
+		document:    `{a: {thing: one, cat: frog}, b: {missing: two, thing: two}}`,
+		expression:  `.a *? .b`,
+		expected: []string{
+			"D0, P[a], (!!map)::{thing: two, cat: frog}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,
+		expression: `.a *? .b`,
+		expected: []string{
+			"D0, P[a], (!!seq)::[{thing: two}]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: {array: [1]}, b: {}}`,
+		expression: `.b *+ .a`,
+		expected: []string{
+			"D0, P[b], (!!map)::{array: [1]}\n",
+		},
+	},
 	{
 		description: "Merge, appending arrays",
 		document:    `{a: {array: [1, 2, animal: dog], value: coconut}, b: {array: [3, 4, animal: cat], value: banana}}`,
@@ -115,6 +160,14 @@ b:
 			"D0, P[a], (!!map)::{array: [1, 2, {animal: dog}, 3, 4, {animal: cat}], value: banana}\n",
 		},
 	},
+	{
+		description: "Merge, only existing fields, appending arrays",
+		document:    `{a: {thing: [1,2]}, b: {thing: [3,4], another: [1]}}`,
+		expression:  `.a *?+ .b`,
+		expected: []string{
+			"D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n",
+		},
+	},
 	{
 		description: "Merge to prefix an element",
 		document:    `{a: cat, b: dog}`,

pkg/yqlib/operator_recursive_descent.go

@@ -7,14 +7,14 @@ import (
 )
 
 type recursiveDescentPreferences struct {
-	TraversePreferences *traversePreferences
+	TraversePreferences traversePreferences
 	RecurseArray        bool
 }
 
 func recursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, expressionNode *ExpressionNode) (*list.List, error) {
 	var results = list.New()
 
-	preferences := expressionNode.Operation.Preferences.(*recursiveDescentPreferences)
+	preferences := expressionNode.Operation.Preferences.(recursiveDescentPreferences)
 	err := recursiveDecent(d, results, matchMap, preferences)
 	if err != nil {
 		return nil, err
@@ -23,7 +23,7 @@ func recursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, express
 	return results, nil
 }
 
-func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List, preferences *recursiveDescentPreferences) error {
+func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List, preferences recursiveDescentPreferences) error {
 	for el := matchMap.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 

pkg/yqlib/operator_split_document.go

@@ -0,0 +1,18 @@
+package yqlib
+
+import (
+	"container/list"
+)
+
+func splitDocumentOperator(d *dataTreeNavigator, matchMap *list.List, expressionNode *ExpressionNode) (*list.List, error) {
+	log.Debugf("-- splitDocumentOperator")
+
+	var index uint = 0
+	for el := matchMap.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidate.Document = index
+		index = index + 1
+	}
+
+	return matchMap, nil
+}

pkg/yqlib/operator_split_document_test.go

@@ -0,0 +1,32 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var splitDocOperatorScenarios = []expressionScenario{
+	{
+		description: "Split empty",
+		document:    ``,
+		expression:  `splitDoc`,
+		expected: []string{
+			"D0, P[], (!!null)::\n",
+		},
+	},
+	{
+		description: "Split array",
+		document:    `[{a: cat}, {b: dog}]`,
+		expression:  `.[] | splitDoc`,
+		expected: []string{
+			"D0, P[0], (!!map)::{a: cat}\n",
+			"D1, P[1], (!!map)::{b: dog}\n",
+		},
+	},
+}
+
+func TestSplitDocOperatorScenarios(t *testing.T) {
+	for _, tt := range splitDocOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Split into Documents", splitDocOperatorScenarios)
+}

pkg/yqlib/operator_strings.go

@@ -0,0 +1,96 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+func joinStringOperator(d *dataTreeNavigator, matchMap *list.List, expressionNode *ExpressionNode) (*list.List, error) {
+	log.Debugf("-- joinStringOperator")
+	joinStr := ""
+
+	rhs, err := d.GetMatchingNodes(matchMap, expressionNode.Rhs)
+	if err != nil {
+		return nil, err
+	}
+	if rhs.Front() != nil {
+		joinStr = rhs.Front().Value.(*CandidateNode).Node.Value
+	}
+
+	var results = list.New()
+
+	for el := matchMap.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		node := unwrapDoc(candidate.Node)
+		if node.Kind != yaml.SequenceNode {
+			return nil, fmt.Errorf("Cannot join with %v, can only join arrays of scalars", node.Tag)
+		}
+		targetNode := join(node.Content, joinStr)
+		result := candidate.CreateChild(nil, targetNode)
+		results.PushBack(result)
+	}
+
+	return results, nil
+}
+
+func join(content []*yaml.Node, joinStr string) *yaml.Node {
+	var stringsToJoin []string
+	for _, node := range content {
+		str := node.Value
+		if node.Tag == "!!null" {
+			str = ""
+		}
+		stringsToJoin = append(stringsToJoin, str)
+	}
+
+	return &yaml.Node{Kind: yaml.ScalarNode, Value: strings.Join(stringsToJoin, joinStr), Tag: "!!str"}
+}
+
+func splitStringOperator(d *dataTreeNavigator, matchMap *list.List, expressionNode *ExpressionNode) (*list.List, error) {
+	log.Debugf("-- splitStringOperator")
+	splitStr := ""
+
+	rhs, err := d.GetMatchingNodes(matchMap, expressionNode.Rhs)
+	if err != nil {
+		return nil, err
+	}
+	if rhs.Front() != nil {
+		splitStr = rhs.Front().Value.(*CandidateNode).Node.Value
+	}
+
+	var results = list.New()
+
+	for el := matchMap.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		node := unwrapDoc(candidate.Node)
+		if node.Tag == "!!null" {
+			continue
+		}
+		if node.Tag != "!!str" {
+			return nil, fmt.Errorf("Cannot split %v, can only split strings", node.Tag)
+		}
+		targetNode := split(node.Value, splitStr)
+		result := candidate.CreateChild(nil, targetNode)
+		results.PushBack(result)
+	}
+
+	return results, nil
+}
+
+func split(value string, spltStr string) *yaml.Node {
+	var contents []*yaml.Node
+
+	if value != "" {
+		var newStrings = strings.Split(value, spltStr)
+		contents = make([]*yaml.Node, len(newStrings))
+
+		for index, str := range newStrings {
+			contents[index] = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: str}
+		}
+	}
+
+	return &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq", Content: contents}
+}

pkg/yqlib/operator_strings_test.go

@@ -0,0 +1,52 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var stringsOperatorScenarios = []expressionScenario{
+	{
+		description: "Join strings",
+		document:    `[cat, meow, 1, null, true]`,
+		expression:  `join("; ")`,
+		expected: []string{
+			"D0, P[], (!!str)::cat; meow; 1; ; true\n",
+		},
+	},
+	{
+		description: "Split strings",
+		document:    `"cat; meow; 1; ; true"`,
+		expression:  `split("; ")`,
+		expected: []string{
+			"D0, P[], (!!seq)::- cat\n- meow\n- \"1\"\n- \"\"\n- \"true\"\n",
+		},
+	},
+	{
+		description: "Split strings one match",
+		document:    `"word"`,
+		expression:  `split("; ")`,
+		expected: []string{
+			"D0, P[], (!!seq)::- word\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `""`,
+		expression: `split("; ")`,
+		expected: []string{
+			"D0, P[], (!!seq)::[]\n", // dont actually want this, just not to error
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `split("; ")`,
+		expected:   []string{},
+	},
+}
+
+func TestStringsOperatorScenarios(t *testing.T) {
+	for _, tt := range stringsOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "String Operators", stringsOperatorScenarios)
+}

pkg/yqlib/operator_traverse_path.go

@@ -10,11 +10,12 @@ import (
 )
 
 type traversePreferences struct {
-	FollowAlias    bool
-	IncludeMapKeys bool
+	DontFollowAlias bool
+	IncludeMapKeys  bool
+	DontAutoCreate  bool // by default, we automatically create entries on the fly.
 }
 
-func splat(d *dataTreeNavigator, matches *list.List, prefs *traversePreferences) (*list.List, error) {
+func splat(d *dataTreeNavigator, matches *list.List, prefs traversePreferences) (*list.List, error) {
 	return traverseNodesWithArrayIndices(matches, make([]*yaml.Node, 0), prefs)
 }
 
@@ -54,8 +55,7 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Oper
 	switch value.Kind {
 	case yaml.MappingNode:
 		log.Debug("its a map with %v entries", len(value.Content)/2)
-		prefs := &traversePreferences{FollowAlias: true}
-		return traverseMap(matchingNode, operation.StringValue, prefs, false)
+		return traverseMap(matchingNode, operation.StringValue, operation.Preferences.(traversePreferences), false)
 
 	case yaml.SequenceNode:
 		log.Debug("its a sequence of %v things!", len(value.Content))
@@ -83,11 +83,10 @@ func traverseArrayOperator(d *dataTreeNavigator, matchingNodes *list.List, expre
 	}
 
 	var indicesToTraverse = rhs.Front().Value.(*CandidateNode).Node.Content
-	prefs := &traversePreferences{FollowAlias: true}
-	return traverseNodesWithArrayIndices(matchingNodes, indicesToTraverse, prefs)
+	return traverseNodesWithArrayIndices(matchingNodes, indicesToTraverse, traversePreferences{})
 }
 
-func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, prefs *traversePreferences) (*list.List, error) {
+func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, prefs traversePreferences) (*list.List, error) {
 	var matchingNodeMap = list.New()
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
@@ -101,7 +100,7 @@ func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse [
 	return matchingNodeMap, nil
 }
 
-func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, prefs *traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
+func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, prefs traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
 	node := matchingNode.Node
 	if node.Tag == "!!null" {
 		log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
@@ -124,7 +123,7 @@ func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml
 	return list.New(), nil
 }
 
-func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs *traversePreferences) (*list.List, error) {
+func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs traversePreferences) (*list.List, error) {
 	if len(indices) == 0 {
 		return traverseMap(candidate, "", prefs, true)
 	}
@@ -188,7 +187,7 @@ func keyMatches(key *yaml.Node, wantedKey string) bool {
 	return matchKey(key.Value, wantedKey)
 }
 
-func traverseMap(matchingNode *CandidateNode, key string, prefs *traversePreferences, splat bool) (*list.List, error) {
+func traverseMap(matchingNode *CandidateNode, key string, prefs traversePreferences, splat bool) (*list.List, error) {
 	var newMatches = orderedmap.NewOrderedMap()
 	err := doTraverseMap(newMatches, matchingNode, key, prefs, splat)
 
@@ -196,7 +195,7 @@ func traverseMap(matchingNode *CandidateNode, key string, prefs *traversePrefere
 		return nil, err
 	}
 
-	if newMatches.Len() == 0 {
+	if !prefs.DontAutoCreate && newMatches.Len() == 0 {
 		//no matches, create one automagically
 		valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}
 		node := matchingNode.Node
@@ -214,7 +213,7 @@ func traverseMap(matchingNode *CandidateNode, key string, prefs *traversePrefere
 	return results, nil
 }
 
-func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, prefs *traversePreferences, splat bool) error {
+func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, prefs traversePreferences, splat bool) error {
 	// value.Content is a concatenated array of key, value,
 	// so keys are in the even indexes, values in odd.
 	// merge aliases are defined first, but we only want to traverse them
@@ -229,7 +228,7 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode,
 
 		log.Debug("checking %v (%v)", key.Value, key.Tag)
 		//skip the 'merge' tag, find a direct match first
-		if key.Tag == "!!merge" && prefs.FollowAlias {
+		if key.Tag == "!!merge" && !prefs.DontFollowAlias {
 			log.Debug("Merge anchor")
 			err := traverseMergeAnchor(newMatches, candidate, value, wantedKey, prefs, splat)
 			if err != nil {
@@ -249,7 +248,7 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode,
 	return nil
 }
 
-func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, prefs *traversePreferences, splat bool) error {
+func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, prefs traversePreferences, splat bool) error {
 	switch value.Kind {
 	case yaml.AliasNode:
 		candidateNode := originalCandidate.CreateChild(nil, value.Alias)

pkg/yqlib/operators.go

@@ -35,14 +35,16 @@ func nodeToMap(candidate *CandidateNode) *list.List {
 	return elMap
 }
 
-func createTraversalTree(path []interface{}) *ExpressionNode {
+func createTraversalTree(path []interface{}, traversePrefs traversePreferences) *ExpressionNode {
 	if len(path) == 0 {
 		return &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
 	} else if len(path) == 1 {
-		return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
+		return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Preferences: traversePrefs, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
 	}
+
 	return &ExpressionNode{
 		Operation: &Operation{OperationType: shortPipeOpType},
-		Lhs:       createTraversalTree(path[0:1]),
-		Rhs:       createTraversalTree(path[1:])}
+		Lhs:       createTraversalTree(path[0:1], traversePrefs),
+		Rhs:       createTraversalTree(path[1:], traversePrefs),
+	}
 }

pkg/yqlib/operators_test.go

@@ -55,7 +55,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 		candidateNode := &CandidateNode{
 			Document:  0,
 			Filename:  "",
-			Node:      &yaml.Node{Tag: "!!null"},
+			Node:      &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode},
 			FileIndex: 0,
 		}
 		inputs.PushBack(candidateNode)
@@ -245,7 +245,7 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt
 		candidateNode := &CandidateNode{
 			Document:  0,
 			Filename:  "",
-			Node:      &yaml.Node{Tag: "!!null"},
+			Node:      &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode},
 			FileIndex: 0,
 		}
 		inputs.PushBack(candidateNode)

pkg/yqlib/stream_evaluator.go

@@ -35,7 +35,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error
 	candidateNode := &CandidateNode{
 		Document:  0,
 		Filename:  "",
-		Node:      &yaml.Node{Tag: "!!null"},
+		Node:      &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode},
 		FileIndex: 0,
 	}
 	inputList := list.New()

pkg/yqlib/utils.go

@@ -36,10 +36,11 @@ func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List
 			return nil, errorReading
 		}
 		candidateNode := &CandidateNode{
-			Document:  currentIndex,
-			Filename:  filename,
-			Node:      &dataBucket,
-			FileIndex: fileIndex,
+			Document:         currentIndex,
+			Filename:         filename,
+			Node:             &dataBucket,
+			FileIndex:        fileIndex,
+			EvaluateTogether: true,
 		}
 
 		inputList.PushBack(candidateNode)

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.3.2'
+version: '4.4.1'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.