Refactored doc generation, add fi fileIndex alias

.github/ISSUE_TEMPLATE/bug_report_v3.md

@@ -10,6 +10,8 @@ assignees: ''
 **Describe the bug**
 A clear and concise description of what the bug is.
 
+Note that any how to questions should be posted in the discussion board and not raised as an issue.
+
 Version of yq: 3.X.X
 Operating system: mac/linux/windows/....
 Installed via: docker/binary release/homebrew/snap/...

.github/ISSUE_TEMPLATE/bug_report_v4.md

@@ -10,6 +10,8 @@ assignees: ''
 **Describe the bug**
 A clear and concise description of what the bug is.
 
+Note that any how to questions should be posted in the discussion board and not raised as an issue.
+
 Version of yq: 4.X.X
 Operating system: mac/linux/windows/....
 Installed via: docker/binary release/homebrew/snap/...

.github/ISSUE_TEMPLATE/feature_request.md

@@ -9,9 +9,11 @@ assignees: ''
 
 **Please describe your feature request.**
 A clear and concise description of what the request is and what it would solve. 
-Ex. I wish I could use yq to [...]
+Eg. I wish I could use yq to [...]
 
-Please note that V3 will no longer have any enhancements.
+Note:
+- how to questions should be posted in the discussion board and not raised as an issue.
+- V3 will no longer have any enhancements.
 
 **Describe the solution you'd like**
 If we have data1.yml like:

README.md

@@ -19,6 +19,14 @@ If you've been using v3 and want/need to upgrade, checkout the [upgrade guide](h
 ### wget
 Use wget to download the pre-compiled binaries:
 
+#### Compressed via tar.gz
+```bash
+wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY}.tar.gz -O - |\
+  tar xz && mv ${BINARY} /usr/bin/yq
+```
+
+#### Plain binary
+
 ```bash
 wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/bin/yq &&\
     chmod +x /usr/bin/yq

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.2.0"
+	Version = "4.2.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.2.0
+FROM mikefarah/yq:4.2.1
 
 COPY entrypoint.sh /entrypoint.sh
 

pkg/yqlib/candidate_node.go

@@ -40,6 +40,7 @@ func (n *CandidateNode) Copy() (*CandidateNode, error) {
 
 // updates this candidate from the given candidate node
 func (n *CandidateNode) UpdateFrom(other *CandidateNode) {
+
 	n.UpdateAttributesFrom(other)
 	n.Node.Content = other.Node.Content
 	n.Node.Value = other.Node.Value

pkg/yqlib/doc/Assign.md

@@ -34,6 +34,27 @@ a:
   g: foof
 ```
 
+## Update node from another file
+Note this will also work when the second file is a scalar (string/number)
+
+Given a sample.yml file of:
+```yaml
+a: apples
+```
+And another sample another.yml file of:
+```yaml
+b: bob
+```
+then
+```bash
+yq eval-all 'select(fileIndex==0).a = select(fileIndex==1) | select(fileIndex==0)' sample.yml another.yml
+```
+will output
+```yaml
+a:
+  b: bob
+```
+
 ## Update node to be the sibling value
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/File Operators.md

@@ -1,9 +1,11 @@
 File operators are most often used with merge when needing to merge specific files together. Note that when doing this, you will need to use `eval-all` to ensure all yaml documents are loaded into memory before performing the merge (as opposed to `eval` which runs the expression once per document).
 
+Note that the `fileIndex` operator has a short alias of `fi`.
+
 ## Merging files
 Note the use of eval-all to ensure all documents are loaded into memory.
 ```bash
-yq eval-all 'select(fileIndex == 0) * select(filename == "file2.yaml")' file1.yaml file2.yaml
+yq eval-all 'select(fi == 0) * select(filename == "file2.yaml")' file1.yaml file2.yaml
 ```
 ## Get filename
 Given a sample.yml file of:
@@ -16,7 +18,7 @@ yq eval 'filename' sample.yml
 ```
 will output
 ```yaml
-sample.yaml
+sample.yml
 ```
 
 ## Get file index
@@ -33,3 +35,17 @@ will output
 0
 ```
 
+## Get file index alias
+Given a sample.yml file of:
+```yaml
+a: cat
+```
+then
+```bash
+yq eval 'fi' sample.yml
+```
+will output
+```yaml
+0
+```
+

pkg/yqlib/doc/headers/File Operators.md

@@ -1,7 +1,9 @@
 File operators are most often used with merge when needing to merge specific files together. Note that when doing this, you will need to use `eval-all` to ensure all yaml documents are loaded into memory before performing the merge (as opposed to `eval` which runs the expression once per document).
 
+Note that the `fileIndex` operator has a short alias of `fi`.
+
 ## Merging files
 Note the use of eval-all to ensure all documents are loaded into memory.
 ```bash
-yq eval-all 'select(fileIndex == 0) * select(filename == "file2.yaml")' file1.yaml file2.yaml
+yq eval-all 'select(fi == 0) * select(filename == "file2.yaml")' file1.yaml file2.yaml
 ```

pkg/yqlib/operator_assign.go

@@ -33,7 +33,9 @@ func AssignUpdateOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNo
 		first := rhs.Front()
 
 		if first != nil {
-			candidate.UpdateFrom(first.Value.(*CandidateNode))
+			rhsCandidate := first.Value.(*CandidateNode)
+			rhsCandidate.Node = UnwrapDoc(rhsCandidate.Node)
+			candidate.UpdateFrom(rhsCandidate)
 		}
 	}
 	// // if there was nothing given, perhaps we are creating a new yaml doc

pkg/yqlib/operator_assign_test.go

@@ -20,6 +20,16 @@ var assignOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: {g: foof}}\n",
 		},
 	},
+	{
+		description:    "Update node from another file",
+		subdescription: "Note this will also work when the second file is a scalar (string/number)",
+		document:       `{a: apples}`,
+		document2:      "{b: bob}",
+		expression:     `select(fileIndex==0).a = select(fileIndex==1) | select(fileIndex==0)`,
+		expected: []string{
+			"D0, P[], (doc)::{a: {b: bob}}\n",
+		},
+	},
 	{
 		description: "Update node to be the sibling value",
 		document:    `{a: {b: child}, b: sibling}`,

pkg/yqlib/operator_file_test.go

@@ -21,6 +21,14 @@ var fileOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!int)::0\n",
 		},
 	},
+	{
+		description: "Get file index alias",
+		document:    `{a: cat}`,
+		expression:  `fi`,
+		expected: []string{
+			"D0, P[], (!!int)::0\n",
+		},
+	},
 }
 
 func TestFileOperatorsScenarios(t *testing.T) {

pkg/yqlib/operators_test.go

@@ -18,6 +18,7 @@ type expressionScenario struct {
 	description           string
 	subdescription        string
 	document              string
+	document2             string
 	expression            string
 	expected              []string
 	skipDoc               bool
@@ -41,6 +42,14 @@ func testScenario(t *testing.T, s *expressionScenario) {
 			t.Error(err, s.document, s.expression)
 			return
 		}
+		if s.document2 != "" {
+			moreInputs, err := readDocuments(strings.NewReader(s.document2), "another.yml", 1)
+			if err != nil {
+				t.Error(err, s.document, s.expression)
+				return
+			}
+			inputs.PushBackList(moreInputs)
+		}
 	} else {
 		candidateNode := &CandidateNode{
 			Document:  0,
@@ -92,7 +101,7 @@ func copyFromHeader(title string, out *os.File) error {
 	return err
 }
 
-func formatYaml(yaml string) string {
+func formatYaml(yaml string, filename string) string {
 	var output bytes.Buffer
 	printer := NewPrinter(bufio.NewWriter(&output), false, true, false, 2, true)
 
@@ -101,7 +110,7 @@ func formatYaml(yaml string) string {
 		panic(err)
 	}
 	streamEvaluator := NewStreamEvaluator()
-	err = streamEvaluator.Evaluate("sample.yaml", strings.NewReader(yaml), node, printer)
+	err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer)
 	if err != nil {
 		panic(err)
 	}
@@ -128,63 +137,116 @@ func documentScenarios(t *testing.T, title string, scenarios []expressionScenari
 
 	for _, s := range scenarios {
 		if !s.skipDoc {
-
-			writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-			if s.subdescription != "" {
-				writeOrPanic(w, s.subdescription)
-				writeOrPanic(w, "\n\n")
-			}
-			formattedDoc := ""
-			if s.document != "" {
-				if s.dontFormatInputForDoc {
-					formattedDoc = s.document + "\n"
-				} else {
-					formattedDoc = formatYaml(s.document)
-				}
-				//TODO: pretty here
-				writeOrPanic(w, "Given a sample.yml file of:\n")
-
-				writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n", formattedDoc))
-				writeOrPanic(w, "then\n")
-				if s.expression != "" {
-					writeOrPanic(w, fmt.Sprintf("```bash\nyq eval '%v' sample.yml\n```\n", s.expression))
-				} else {
-					writeOrPanic(w, "```bash\nyq eval sample.yml\n```\n")
-				}
-			} else {
-				writeOrPanic(w, "Running\n")
-				writeOrPanic(w, fmt.Sprintf("```bash\nyq eval --null-input '%v'\n```\n", s.expression))
-			}
-
-			writeOrPanic(w, "will output\n")
-
-			var output bytes.Buffer
-			var err error
-			printer := NewPrinter(bufio.NewWriter(&output), false, true, false, 2, true)
-			streamEvaluator := NewStreamEvaluator()
-
-			if s.document != "" {
-				node, err := treeCreator.ParsePath(s.expression)
-				if err != nil {
-					t.Error(err, s.expression)
-				}
-				err = streamEvaluator.Evaluate("sample.yaml", strings.NewReader(formattedDoc), node, printer)
-
-				if err != nil {
-					t.Error(err, s.expression)
-				}
-			} else {
-				err = streamEvaluator.EvaluateNew(s.expression, printer)
-				if err != nil {
-					t.Error(err, s.expression)
-				}
-			}
-
-			writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", output.String()))
-
+			documentScenario(t, w, s)
 		}
-
 	}
 	w.Flush()
 }
+
+func documentScenario(t *testing.T, w *bufio.Writer, s expressionScenario) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	formattedDoc, formattedDoc2 := documentInput(w, s)
+
+	writeOrPanic(w, "will output\n")
+
+	documentOutput(t, w, s, formattedDoc, formattedDoc2)
+}
+
+func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
+	formattedDoc := ""
+	formattedDoc2 := ""
+	command := "eval"
+	if s.document != "" {
+		if s.dontFormatInputForDoc {
+			formattedDoc = s.document + "\n"
+		} else {
+			formattedDoc = formatYaml(s.document, "sample.yml")
+		}
+
+		writeOrPanic(w, "Given a sample.yml file of:\n")
+		writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n", formattedDoc))
+
+		files := "sample.yml"
+
+		if s.document2 != "" {
+			if s.dontFormatInputForDoc {
+				formattedDoc2 = s.document2 + "\n"
+			} else {
+				formattedDoc2 = formatYaml(s.document2, "another.yml")
+			}
+
+			writeOrPanic(w, "And another sample another.yml file of:\n")
+			writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n", formattedDoc2))
+			files = "sample.yml another.yml"
+			command = "eval-all"
+		}
+
+		writeOrPanic(w, "then\n")
+		if s.expression != "" {
+			writeOrPanic(w, fmt.Sprintf("```bash\nyq %v '%v' %v\n```\n", command, s.expression, files))
+		} else {
+			writeOrPanic(w, fmt.Sprintf("```bash\nyq %v %v\n```\n", command, files))
+		}
+	} else {
+		writeOrPanic(w, "Running\n")
+		writeOrPanic(w, fmt.Sprintf("```bash\nyq %v --null-input '%v'\n```\n", command, s.expression))
+	}
+	return formattedDoc, formattedDoc2
+}
+
+func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) {
+	var output bytes.Buffer
+	var err error
+	printer := NewPrinter(bufio.NewWriter(&output), false, true, false, 2, true)
+
+	node, err := treeCreator.ParsePath(s.expression)
+	if err != nil {
+		t.Error(fmt.Errorf("Error parsing expression %v of %v: %v", s.expression, s.description, err))
+		return
+	}
+
+	inputs := list.New()
+
+	if s.document != "" {
+		inputs, err = readDocuments(strings.NewReader(formattedDoc), "sample.yml", 0)
+		if err != nil {
+			t.Error(err, s.document, s.expression)
+			return
+		}
+		if s.document2 != "" {
+			moreInputs, err := readDocuments(strings.NewReader(formattedDoc2), "another.yml", 1)
+			if err != nil {
+				t.Error(err, s.document, s.expression)
+				return
+			}
+			inputs.PushBackList(moreInputs)
+		}
+	} else {
+		candidateNode := &CandidateNode{
+			Document:  0,
+			Filename:  "",
+			Node:      &yaml.Node{Tag: "!!null"},
+			FileIndex: 0,
+		}
+		inputs.PushBack(candidateNode)
+
+	}
+
+	results, err := treeNavigator.GetMatchingNodes(inputs, node)
+	if err != nil {
+		t.Error(err, s.expression)
+	}
+
+	err = printer.PrintResults(results)
+	if err != nil {
+		t.Error(err, s.expression)
+	}
+
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", output.String()))
+}

pkg/yqlib/path_tokeniser.go

@@ -200,6 +200,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`alias`), opAssignableToken(GetAlias, AssignAlias))
 	lexer.Add([]byte(`filename`), opToken(GetFilename))
 	lexer.Add([]byte(`fileIndex`), opToken(GetFileIndex))
+	lexer.Add([]byte(`fi`), opToken(GetFileIndex))
 	lexer.Add([]byte(`path`), opToken(GetPath))
 
 	lexer.Add([]byte(`lineComment`), opTokenWithPrefs(GetComment, AssignComment, &CommentOpPreferences{LineComment: true}))

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.2.0'
+version: '4.2.1'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.