refactored

pkg/yqlib/candidate_node.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"fmt"

pkg/yqlib/data_tree_navigator.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"fmt"

pkg/yqlib/data_tree_navigator_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"container/list"

pkg/yqlib/doc/Equal Operator.md

@@ -0,0 +1,64 @@
+# Equal Operator
+## Examples
+### Example 0
+sample.yml:
+```yaml
+[cat,goat,dog]
+```
+Expression
+```bash
+yq '.[] | (. == "*at")' < sample.yml
+```
+Result
+```yaml
+true
+true
+false
+```
+### Example 1
+sample.yml:
+```yaml
+[3, 4, 5]
+```
+Expression
+```bash
+yq '.[] | (. == 4)' < sample.yml
+```
+Result
+```yaml
+false
+true
+false
+```
+### Example 2
+sample.yml:
+```yaml
+a: { cat: {b: apple, c: whatever}, pat: {b: banana} }
+```
+Expression
+```bash
+yq '.a | (.[].b == "apple")' < sample.yml
+```
+Result
+```yaml
+true
+false
+```
+### Example 3
+Expression
+```bash
+yq 'null == null' < sample.yml
+```
+Result
+```yaml
+true
+```
+### Example 4
+Expression
+```bash
+yq 'null == ~' < sample.yml
+```
+Result
+```yaml
+true
+```

pkg/yqlib/encoder.go

@@ -74,16 +74,14 @@ func mapKeysToStrings(node *yaml.Node) {
 	}
 }
 
-func NewJsonEncoder(destination io.Writer, prettyPrint bool, indent int) Encoder {
+func NewJsonEncoder(destination io.Writer, indent int) Encoder {
 	var encoder = json.NewEncoder(destination)
 	var indentString = ""
 
 	for index := 0; index < indent; index++ {
 		indentString = indentString + " "
 	}
-	if prettyPrint {
-		encoder.SetIndent("", indentString)
-	}
+	encoder.SetIndent("", indentString)
 	return &jsonEncoder{encoder}
 }
 

pkg/yqlib/lib.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"bytes"
@@ -9,8 +9,6 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-var log = logging.MustGetLogger("yq-treeops")
-
 type OperationType struct {
 	Type       string
 	NumArgs    uint // number of arguments to the op
@@ -25,6 +23,9 @@ type OperationType struct {
 // - mergeAppend (merges and appends arrays)
 // - mergeEmpty (sets only if the document is empty, do I do that now?)
 // - updateTag - not recursive
+// - select by tag (tag==)
+// - get tag (tag)
+// - select by style (style==)
 // - compare ??
 // - validate ??
 // - exists

pkg/yqlib/matchKeyString.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 func Match(name string, pattern string) (matched bool) {
 	if pattern == "" {

pkg/yqlib/operator_assign_update.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import "container/list"
 

pkg/yqlib/operator_assign_update_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"testing"

pkg/yqlib/operator_booleans.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"container/list"

pkg/yqlib/operator_booleans_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"testing"

pkg/yqlib/operator_collect.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"container/list"

pkg/yqlib/operator_collect_object.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"container/list"

pkg/yqlib/operator_collect_object_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"testing"

pkg/yqlib/operator_collect_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"testing"

pkg/yqlib/operator_create_map.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"container/list"

pkg/yqlib/operator_create_map_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"testing"

pkg/yqlib/operator_delete.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"container/list"

pkg/yqlib/operator_equals.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"container/list"

pkg/yqlib/operator_equals_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"testing"
@@ -49,4 +49,5 @@ func TestEqualOperatorScenarios(t *testing.T) {
 	for _, tt := range equalsOperatorScenarios {
 		testScenario(t, &tt)
 	}
+	documentScenarios(t, "Equal Operator", equalsOperatorScenarios)
 }

pkg/yqlib/operator_explode.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"container/list"

pkg/yqlib/operator_explode_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"testing"

pkg/yqlib/operator_multilpy.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"fmt"

pkg/yqlib/operator_multiply_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"testing"

pkg/yqlib/operator_not.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import "container/list"
 

pkg/yqlib/operator_not_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"testing"

pkg/yqlib/operator_recursive_descent.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"container/list"

pkg/yqlib/operator_recursive_descent_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"testing"

pkg/yqlib/operator_select.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"container/list"

pkg/yqlib/operator_select_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"testing"

pkg/yqlib/operator_self.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import "container/list"
 

pkg/yqlib/operator_traverse_path.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"fmt"
@@ -186,26 +186,32 @@ func traverseArray(candidate *CandidateNode, operation *Operation) ([]*Candidate
 
 	}
 
-	index := operation.Value.(int64)
-	indexToUse := index
-	contentLength := int64(len(candidate.Node.Content))
-	for contentLength <= index {
-		candidate.Node.Content = append(candidate.Node.Content, &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"})
-		contentLength = int64(len(candidate.Node.Content))
-	}
+	switch operation.Value.(type) {
+	case int64:
+		index := operation.Value.(int64)
+		indexToUse := index
+		contentLength := int64(len(candidate.Node.Content))
+		for contentLength <= index {
+			candidate.Node.Content = append(candidate.Node.Content, &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"})
+			contentLength = int64(len(candidate.Node.Content))
+		}
 
-	if indexToUse < 0 {
-		indexToUse = contentLength + indexToUse
-	}
+		if indexToUse < 0 {
+			indexToUse = contentLength + indexToUse
+		}
 
-	if indexToUse < 0 {
-		return nil, fmt.Errorf("Index [%v] out of range, array size is %v", index, contentLength)
-	}
+		if indexToUse < 0 {
+			return nil, fmt.Errorf("Index [%v] out of range, array size is %v", index, contentLength)
+		}
 
-	return []*CandidateNode{&CandidateNode{
-		Node:     candidate.Node.Content[indexToUse],
-		Document: candidate.Document,
-		Path:     append(candidate.Path, index),
-	}}, nil
+		return []*CandidateNode{&CandidateNode{
+			Node:     candidate.Node.Content[indexToUse],
+			Document: candidate.Document,
+			Path:     append(candidate.Path, index),
+		}}, nil
+	default:
+		log.Debug("argument not an int (%v), no array matches", operation.Value)
+		return nil, nil
+	}
 
 }

pkg/yqlib/operator_traverse_path_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"testing"
@@ -126,6 +126,26 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[b c], (!!str)::frog\n",
 		},
 	},
+	{
+		document:   `[1,2,3]`,
+		expression: `.b`,
+		expected:   []string{},
+	},
+	{
+		document:   `[1,2,3]`,
+		expression: `[0]`,
+		expected: []string{
+			"D0, P[0], (!!int)::1\n",
+		},
+	},
+	{
+		description: `Maps can have numbers as keys, so this default to a non-exisiting key behaviour.`,
+		document:    `{a: b}`,
+		expression:  `[0]`,
+		expected: []string{
+			"D0, P[0], (!!null)::null\n",
+		},
+	},
 	{
 		document:   mergeDocSample,
 		expression: `.foobar`,

pkg/yqlib/operator_union.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import "container/list"
 

pkg/yqlib/operator_union_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"testing"

pkg/yqlib/operator_value.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import "container/list"
 

pkg/yqlib/operator_value_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"testing"

pkg/yqlib/operators.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"container/list"

pkg/yqlib/operators_test.go

@@ -0,0 +1,85 @@
+package yqlib
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+type expressionScenario struct {
+	description string
+	document    string
+	expression  string
+	expected    []string
+}
+
+func testScenario(t *testing.T, s *expressionScenario) {
+
+	nodes := readDoc(t, s.document)
+	path, errPath := treeCreator.ParsePath(s.expression)
+	if errPath != nil {
+		t.Error(errPath)
+		return
+	}
+	results, errNav := treeNavigator.GetMatchingNodes(nodes, path)
+
+	if errNav != nil {
+		t.Error(errNav)
+		return
+	}
+	test.AssertResultComplexWithContext(t, s.expected, resultsToString(results), fmt.Sprintf("exp: %v\ndoc: %v", s.expression, s.document))
+}
+
+func documentScenarios(t *testing.T, title string, scenarios []expressionScenario) {
+	f, err := os.Create(fmt.Sprintf("doc/%v.md", title))
+
+	if err != nil {
+		panic(err)
+	}
+	defer f.Close()
+	w := bufio.NewWriter(f)
+	w.WriteString(fmt.Sprintf("# %v\n", title))
+	w.WriteString(fmt.Sprintf("## Examples\n"))
+
+	printer := NewPrinter(false, true, false, 2, true)
+
+	for index, s := range scenarios {
+		if s.description != "" {
+			w.WriteString(fmt.Sprintf("### %v\n", s.description))
+		} else {
+			w.WriteString(fmt.Sprintf("### Example %v\n", index))
+		}
+		if s.document != "" {
+			w.WriteString(fmt.Sprintf("sample.yml:\n"))
+			w.WriteString(fmt.Sprintf("```yaml\n%v\n```\n", s.document))
+		}
+		if s.expression != "" {
+			w.WriteString(fmt.Sprintf("Expression\n"))
+			w.WriteString(fmt.Sprintf("```bash\nyq '%v' < sample.yml\n```\n", s.expression))
+		}
+
+		w.WriteString(fmt.Sprintf("Result\n"))
+
+		nodes := readDoc(t, s.document)
+		path, errPath := treeCreator.ParsePath(s.expression)
+		if errPath != nil {
+			t.Error(errPath)
+			return
+		}
+		var output bytes.Buffer
+		results, err := treeNavigator.GetMatchingNodes(nodes, path)
+		printer.PrintResults(results, bufio.NewWriter(&output))
+
+		w.WriteString(fmt.Sprintf("```yaml\n%v```\n", output.String()))
+
+		if err != nil {
+			panic(err)
+		}
+
+	}
+	w.Flush()
+}

pkg/yqlib/operatory_style.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"container/list"

pkg/yqlib/operatory_style_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"testing"

pkg/yqlib/path_parse_test.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"fmt"

pkg/yqlib/path_postfix.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"errors"

pkg/yqlib/path_tokeniser.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import (
 	"fmt"

pkg/yqlib/path_tree.go

@@ -1,4 +1,4 @@
-package treeops
+package yqlib
 
 import "fmt"
 

pkg/yqlib/printer.go

@@ -0,0 +1,72 @@
+package yqlib
+
+import (
+	"bufio"
+	"container/list"
+	"io"
+
+	"gopkg.in/yaml.v3"
+)
+
+type Printer interface {
+	PrintResults(matchingNodes *list.List, writer io.Writer) error
+}
+
+type resultsPrinter struct {
+	outputToJSON       bool
+	unwrapScalar       bool
+	colorsEnabled      bool
+	indent             int
+	printDocSeparators bool
+}
+
+func NewPrinter(outputToJSON bool, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
+	return &resultsPrinter{outputToJSON, unwrapScalar, colorsEnabled, indent, printDocSeparators}
+}
+
+func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
+	var encoder Encoder
+	if node.Kind == yaml.ScalarNode && p.unwrapScalar && !p.outputToJSON {
+		return p.writeString(writer, node.Value+"\n")
+	}
+	if p.outputToJSON {
+		encoder = NewJsonEncoder(writer, p.indent)
+	} else {
+		encoder = NewYamlEncoder(writer, p.indent, p.colorsEnabled)
+	}
+	return encoder.Encode(node)
+}
+
+func (p *resultsPrinter) writeString(writer io.Writer, txt string) error {
+	_, errorWriting := writer.Write([]byte(txt))
+	return errorWriting
+}
+
+func (p *resultsPrinter) PrintResults(matchingNodes *list.List, writer io.Writer) error {
+
+	bufferedWriter := bufio.NewWriter(writer)
+	defer safelyFlush(bufferedWriter)
+
+	if matchingNodes.Len() == 0 {
+		log.Debug("no matching results, nothing to print")
+		return nil
+	}
+
+	var previousDocIndex uint = 0
+
+	for el := matchingNodes.Front(); el != nil; el = el.Next() {
+		mappedDoc := el.Value.(*CandidateNode)
+
+		if previousDocIndex != mappedDoc.Document && p.printDocSeparators {
+			p.writeString(bufferedWriter, "---\n")
+		}
+
+		if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil {
+			return err
+		}
+
+		previousDocIndex = mappedDoc.Document
+	}
+
+	return nil
+}

pkg/yqlib/treeops/operators_test.go

@@ -1,31 +0,0 @@
-package treeops
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/mikefarah/yq/v4/test"
-)
-
-type expressionScenario struct {
-	document   string
-	expression string
-	expected   []string
-}
-
-func testScenario(t *testing.T, s *expressionScenario) {
-
-	nodes := readDoc(t, s.document)
-	path, errPath := treeCreator.ParsePath(s.expression)
-	if errPath != nil {
-		t.Error(errPath)
-		return
-	}
-	results, errNav := treeNavigator.GetMatchingNodes(nodes, path)
-
-	if errNav != nil {
-		t.Error(errNav)
-		return
-	}
-	test.AssertResultComplexWithContext(t, s.expected, resultsToString(results), fmt.Sprintf("exp: %v\ndoc: %v", s.expression, s.document))
-}

pkg/yqlib/treeops/sample.yaml

@@ -1 +0,0 @@
-{name: Mike, pets: [cat, dog]}

pkg/yqlib/treeops/temp.json

@@ -1,7 +0,0 @@
-{
-  "name": "Mike",
-  "pets": [
-    "cat",
-    "dog"
-  ]
-}

pkg/yqlib/utils.go

@@ -0,0 +1,122 @@
+package yqlib
+
+import (
+	"bufio"
+	"container/list"
+	"errors"
+	"io"
+	"os"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func readStream(filename string) (*yaml.Decoder, error) {
+	if filename == "" {
+		return nil, errors.New("Must provide filename")
+	}
+
+	var stream io.Reader
+	if filename == "-" {
+		stream = bufio.NewReader(os.Stdin)
+	} else {
+		file, err := os.Open(filename) // nolint gosec
+		if err != nil {
+			return nil, err
+		}
+		defer safelyCloseFile(file)
+		stream = file
+	}
+	return yaml.NewDecoder(stream), nil
+}
+
+// put this in lib
+func Evaluate(filename string, node *PathTreeNode) (*list.List, error) {
+
+	var treeNavigator = NewDataTreeNavigator(NavigationPrefs{})
+
+	var matchingNodes = list.New()
+
+	var currentIndex uint = 0
+	var decoder, err = readStream(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	for {
+		var dataBucket yaml.Node
+		errorReading := decoder.Decode(&dataBucket)
+
+		if errorReading == io.EOF {
+			return matchingNodes, nil
+		} else if errorReading != nil {
+			return nil, errorReading
+		}
+		candidateNode := &CandidateNode{
+			Document: currentIndex,
+			Filename: filename,
+			Node:     &dataBucket,
+		}
+		inputList := list.New()
+		inputList.PushBack(candidateNode)
+
+		newMatches, errorParsing := treeNavigator.GetMatchingNodes(inputList, node)
+		if errorParsing != nil {
+			return nil, errorParsing
+		}
+		matchingNodes.PushBackList(newMatches)
+		currentIndex = currentIndex + 1
+	}
+
+	return matchingNodes, nil
+}
+
+func safelyRenameFile(from string, to string) {
+	if renameError := os.Rename(from, to); renameError != nil {
+		log.Debugf("Error renaming from %v to %v, attempting to copy contents", from, to)
+		log.Debug(renameError.Error())
+		// can't do this rename when running in docker to a file targeted in a mounted volume,
+		// so gracefully degrade to copying the entire contents.
+		if copyError := copyFileContents(from, to); copyError != nil {
+			log.Errorf("Failed copying from %v to %v", from, to)
+			log.Error(copyError.Error())
+		} else {
+			removeErr := os.Remove(from)
+			if removeErr != nil {
+				log.Errorf("failed removing original file: %s", from)
+			}
+		}
+	}
+}
+
+// thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
+func copyFileContents(src, dst string) (err error) {
+	in, err := os.Open(src) // nolint gosec
+	if err != nil {
+		return err
+	}
+	defer safelyCloseFile(in)
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer safelyCloseFile(out)
+	if _, err = io.Copy(out, in); err != nil {
+		return err
+	}
+	return out.Sync()
+}
+
+func safelyFlush(writer *bufio.Writer) {
+	if err := writer.Flush(); err != nil {
+		log.Error("Error flushing writer!")
+		log.Error(err.Error())
+	}
+
+}
+func safelyCloseFile(file *os.File) {
+	err := file.Close()
+	if err != nil {
+		log.Error("Error closing file!")
+		log.Error(err.Error())
+	}
+}