attempt to fix dockre

json.Encoder and json.Marshal implicitly use HTMLEscape to convert
>, <, &, with \u003c, \u003e, \u0026. This behavior carries over
to yq, where chars will be escaped when outputting json but not when
outputting yaml.

This changeset disables this behavior via encoder.SetEscapeHTML(false).
Unfortunately there is no equivalent option for json.Marshal, so its
single usage has been replaced with an encoder (with escaping disabled).

.github/workflows/release.yml

@@ -3,6 +3,7 @@ on:
   push:
     tags:
       - 'v*'
+      - 'dockerfix'
 
 jobs:
   publishGitRelease:

README.md

@@ -90,7 +90,7 @@ docker run --rm -v "${PWD}":/workdir mikefarah/yq <command> [flags] [expression
 #### Run commands interactively:
 
 ```bash
-docker run --rm -it -v "${PWD}":/workdir mikefarah/yq sh
+docker run --rm -it -v "${PWD}":/workdir --entrypoint sh mikefarah/yq
 ```
 
 It can be useful to have a bash function to avoid typing the whole docker command:

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.4.0"
+	Version = "4.4.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.4.0
+FROM mikefarah/yq:4.4.1
 
 COPY entrypoint.sh /entrypoint.sh
 

pkg/yqlib/candidate_node.go

@@ -15,6 +15,9 @@ type CandidateNode struct {
 	Document  uint          // the document index of this node
 	Filename  string
 	FileIndex int
+	// when performing op against all nodes given, this will treat all the nodes as one
+	// (e.g. top level cross document merge). This property does not propegate to child nodes.
+	EvaluateTogether bool
 }
 
 func (n *CandidateNode) GetKey() string {

pkg/yqlib/doc/Add.md

@@ -209,3 +209,15 @@ will output
 a: 4
 ```
 
+## Add to null
+Adding to null simply returns the rhs
+
+Running
+```bash
+yq eval --null-input 'null + "cat"'
+```
+will output
+```yaml
+cat
+```
+

pkg/yqlib/doc/Multiply.md

@@ -1,13 +1,14 @@
-Like the multiple operator in `jq`, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
+Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
 
-Upcoming versions of `yq` will add support for other types of multiplication (numbers, strings).
-
-To concatenate when merging objects, use the `*+` form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
+To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
 
+To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?.
 Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
 
+Multiplication of strings and numbers are not yet supported.
+
 ## Merging files
-Note the use of eval-all to ensure all documents are loaded into memory.
+Note the use of `eval-all` to ensure all documents are loaded into memory.
 
 ```bash
 yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml

pkg/yqlib/doc/headers/Multiply.md

@@ -1,13 +1,14 @@
-Like the multiple operator in `jq`, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
+Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
 
-Upcoming versions of `yq` will add support for other types of multiplication (numbers, strings).
-
-To concatenate when merging objects, use the `*+` form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
+To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
 
+To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?.
 Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
 
+Multiplication of strings and numbers are not yet supported.
+
 ## Merging files
-Note the use of eval-all to ensure all documents are loaded into memory.
+Note the use of `eval-all` to ensure all documents are loaded into memory.
 
 ```bash
 yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml

pkg/yqlib/encoder.go

@@ -76,6 +76,8 @@ func mapKeysToStrings(node *yaml.Node) {
 
 func NewJsonEncoder(destination io.Writer, indent int) Encoder {
 	var encoder = json.NewEncoder(destination)
+	encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
+
 	var indentString = ""
 
 	for index := 0; index < indent; index++ {
@@ -153,11 +155,15 @@ func (o *orderedMap) UnmarshalJSON(data []byte) error {
 }
 
 func (o orderedMap) MarshalJSON() ([]byte, error) {
-	if o.kv == nil {
-		return json.Marshal(o.altVal)
-	}
 	buf := new(bytes.Buffer)
 	enc := json.NewEncoder(buf)
+	enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
+	if o.kv == nil {
+		if err := enc.Encode(o.altVal); err != nil {
+			return nil, err
+		}
+		return buf.Bytes(), nil
+	}
 	buf.WriteByte('{')
 	for idx, el := range o.kv {
 		if err := enc.Encode(el.K); err != nil {

pkg/yqlib/encoder_test.go

@@ -9,29 +9,11 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-var sampleYaml = `zabbix: winner
-apple: great
-banana:
-- {cobra: kai, angus: bob}
-`
-
-var expectedJson = `{
-  "zabbix": "winner",
-  "apple": "great",
-  "banana": [
-    {
-      "cobra": "kai",
-      "angus": "bob"
-    }
-  ]
-}
-`
-
-func TestJsonEncoderPreservesObjectOrder(t *testing.T) {
+func yamlToJson(sampleYaml string, indent int) string {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
-	var jsonEncoder = NewJsonEncoder(writer, 2)
+	var jsonEncoder = NewJsonEncoder(writer, indent)
 	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0)
 	if err != nil {
 		panic(err)
@@ -42,6 +24,33 @@ func TestJsonEncoderPreservesObjectOrder(t *testing.T) {
 		panic(err)
 	}
 	writer.Flush()
-	test.AssertResult(t, expectedJson, output.String())
 
+	return strings.TrimSuffix(output.String(), "\n")
+}
+
+func TestJsonEncoderPreservesObjectOrder(t *testing.T) {
+	var sampleYaml = `zabbix: winner
+apple: great
+banana:
+- {cobra: kai, angus: bob}
+`
+	var expectedJson = `{
+  "zabbix": "winner",
+  "apple": "great",
+  "banana": [
+    {
+      "cobra": "kai",
+      "angus": "bob"
+    }
+  ]
+}`
+	var actualJson = yamlToJson(sampleYaml, 2)
+	test.AssertResult(t, expectedJson, actualJson)
+}
+
+func TestJsonEncoderDoesNotEscapeHTMLChars(t *testing.T) {
+	var sampleYaml = `build: "( ./lint && ./format && ./compile ) < src.code"`
+	var expectedJson = `{"build":"( ./lint && ./format && ./compile ) < src.code"}`
+	var actualJson = yamlToJson(sampleYaml, 0)
+	test.AssertResult(t, expectedJson, actualJson)
 }

pkg/yqlib/operator_add.go

@@ -47,9 +47,14 @@ func add(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*Candida
 	lhs.Node = unwrapDoc(lhs.Node)
 	rhs.Node = unwrapDoc(rhs.Node)
 
-	target := lhs.CreateChild(nil, &yaml.Node{})
 	lhsNode := lhs.Node
 
+	if lhsNode.Tag == "!!null" {
+		return lhs.CreateChild(nil, rhs.Node), nil
+	}
+
+	target := lhs.CreateChild(nil, &yaml.Node{})
+
 	switch lhsNode.Kind {
 	case yaml.MappingNode:
 		return nil, fmt.Errorf("Maps not yet supported for addition")

pkg/yqlib/operator_add_test.go

@@ -5,6 +5,15 @@ import (
 )
 
 var addOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `[{a: foo, b: bar}, {a: 1, b: 2}]`,
+		expression: ".[] | .a + .b",
+		expected: []string{
+			"D0, P[0 a], (!!str)::foobar\n",
+			"D0, P[1 a], (!!int)::3\n",
+		},
+	},
 	{
 		description: "Concatenate and assign arrays",
 		document:    `{a: {val: thing, b: [cat,dog]}}`,
@@ -103,6 +112,14 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: 4}\n",
 		},
 	},
+	{
+		description:    "Add to null",
+		subdescription: "Adding to null simply returns the rhs",
+		expression:     `null + "cat"`,
+		expected: []string{
+			"D0, P[], (!!str)::cat\n",
+		},
+	},
 }
 
 func TestAddOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_multiply.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"fmt"
+	"strconv"
 
 	"container/list"
 
@@ -10,21 +11,19 @@ import (
 
 type crossFunctionCalculation func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)
 
-func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (*list.List, error) {
-	lhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Lhs)
+func doCrossFunc(d *dataTreeNavigator, contextList *list.List, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (*list.List, error) {
+	var results = list.New()
+	lhs, err := d.GetMatchingNodes(contextList, expressionNode.Lhs)
 	if err != nil {
 		return nil, err
 	}
 	log.Debugf("crossFunction LHS len: %v", lhs.Len())
 
-	rhs, err := d.GetMatchingNodes(matchingNodes, expressionNode.Rhs)
+	rhs, err := d.GetMatchingNodes(contextList, expressionNode.Rhs)
 
 	if err != nil {
 		return nil, err
 	}
-	log.Debugf("crossFunction RHS len: %v", rhs.Len())
-
-	var results = list.New()
 
 	for el := lhs.Front(); el != nil; el = el.Next() {
 		lhsCandidate := el.Value.(*CandidateNode)
@@ -43,6 +42,32 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, expressionNod
 	return results, nil
 }
 
+func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode, calculation crossFunctionCalculation) (*list.List, error) {
+	var results = list.New()
+
+	var evaluateAllTogether = true
+	for matchEl := matchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
+		evaluateAllTogether = evaluateAllTogether && matchEl.Value.(*CandidateNode).EvaluateTogether
+		if !evaluateAllTogether {
+			break
+		}
+	}
+	if evaluateAllTogether {
+		return doCrossFunc(d, matchingNodes, expressionNode, calculation)
+	}
+
+	for matchEl := matchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
+		contextList := nodeToMap(matchEl.Value.(*CandidateNode))
+		innerResults, err := doCrossFunc(d, contextList, expressionNode, calculation)
+		if err != nil {
+			return nil, err
+		}
+		results.PushBackList(innerResults)
+	}
+
+	return results, nil
+}
+
 type multiplyPreferences struct {
 	AppendArrays  bool
 	TraversePrefs traversePreferences
@@ -69,12 +94,31 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, lhs *C
 				return nil, err
 			}
 			return mergeObjects(d, newThing, rhs, preferences)
-
+		} else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" {
+			return multiplyIntegers(lhs, rhs)
 		}
 		return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
 	}
 }
 
+func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	target := lhs.CreateChild(nil, &yaml.Node{})
+	target.Node.Kind = yaml.ScalarNode
+	target.Node.Style = lhs.Node.Style
+	target.Node.Tag = "!!int"
+
+	lhsNum, err := strconv.Atoi(lhs.Node.Value)
+	if err != nil {
+		return nil, err
+	}
+	rhsNum, err := strconv.Atoi(rhs.Node.Value)
+	if err != nil {
+		return nil, err
+	}
+	target.Node.Value = fmt.Sprintf("%v", lhsNum*rhsNum)
+	return target, nil
+}
+
 func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) (*CandidateNode, error) {
 	shouldAppendArrays := preferences.AppendArrays
 	var results = list.New()

pkg/yqlib/operator_multiply_test.go

@@ -13,6 +13,27 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{a: 2, b: 5}`,
+		document2:  `{a: 3, b: 10}`,
+		expression: `.a * .b`,
+		expected: []string{
+			"D0, P[a], (!!int)::10\n",
+			"D0, P[a], (!!int)::20\n",
+			"D0, P[a], (!!int)::15\n",
+			"D0, P[a], (!!int)::30\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: 2}`,
+		document2:  `{b: 10}`,
+		expression: `select(fi ==0) * select(fi==1)`,
+		expected: []string{
+			"D0, P[], (!!map)::{a: 2, b: 10}\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		expression: `{} * {"cat":"dog"}`,
@@ -123,6 +144,14 @@ b:
 			"D0, P[a], (!!seq)::[{thing: two}]\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{a: {array: [1]}, b: {}}`,
+		expression: `.b *+ .a`,
+		expected: []string{
+			"D0, P[b], (!!map)::{array: [1]}\n",
+		},
+	},
 	{
 		description: "Merge, appending arrays",
 		document:    `{a: {array: [1, 2, animal: dog], value: coconut}, b: {array: [3, 4, animal: cat], value: banana}}`,

pkg/yqlib/utils.go

@@ -36,10 +36,11 @@ func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List
 			return nil, errorReading
 		}
 		candidateNode := &CandidateNode{
-			Document:  currentIndex,
-			Filename:  filename,
-			Node:      &dataBucket,
-			FileIndex: fileIndex,
+			Document:         currentIndex,
+			Filename:         filename,
+			Node:             &dataBucket,
+			FileIndex:        fileIndex,
+			EvaluateTogether: true,
 		}
 
 		inputList.PushBack(candidateNode)

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.4.0'
+version: '4.4.1'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.