From 708ff02e8d1ea2e372d8c832a9924edc01879d53 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 13 Nov 2020 13:19:54 +1100 Subject: [PATCH] Fixed collect object for multi doc --- cmd/evaluate_all_command.go | 64 ++++++++++++ cmd/evalute_sequence_command.go | 44 ++++---- cmd/root.go | 2 +- pkg/yqlib/document_index_operator_test.go | 4 +- pkg/yqlib/lib.go | 1 + pkg/yqlib/operator_collect_object.go | 38 ++++++- pkg/yqlib/operator_collect_object_test.go | 37 ++++++- pkg/yqlib/operator_collect_test.go | 14 ++- pkg/yqlib/operator_create_map.go | 70 ++++++++++--- pkg/yqlib/operator_create_map_test.go | 44 ++++++-- pkg/yqlib/operators.go | 4 + pkg/yqlib/operators_test.go | 43 +++++--- pkg/yqlib/path_parse_test.go | 5 + pkg/yqlib/path_postfix.go | 6 ++ pkg/yqlib/path_tokeniser.go | 7 +- pkg/yqlib/printer.go | 25 +++-- pkg/yqlib/utils.go | 116 +++++++++++++++------- 17 files changed, 406 insertions(+), 118 deletions(-) create mode 100644 cmd/evaluate_all_command.go diff --git a/cmd/evaluate_all_command.go b/cmd/evaluate_all_command.go new file mode 100644 index 0000000..e9a717a --- /dev/null +++ b/cmd/evaluate_all_command.go @@ -0,0 +1,64 @@ +package cmd + +import ( + "os" + + "github.com/mikefarah/yq/v4/pkg/yqlib" + "github.com/spf13/cobra" +) + +func createEvaluateAllCommand() *cobra.Command { + var cmdEvalAll = &cobra.Command{ + Use: "eval-all [expression] [yaml_file1]...", + Aliases: []string{"ea"}, + Short: "Loads all yaml documents of all yaml files and runs expression once", + Example: ` +yq es '.a.b | length' file1.yml file2.yml +yq es < sample.yaml +yq es -n '{"a": "b"}' +`, + Long: "Evaluate All:\nUseful when you need to run an expression across several yaml documents or files. Consumes more memory than eval-seq", + RunE: evaluateAll, + } + return cmdEvalAll +} +func evaluateAll(cmd *cobra.Command, args []string) error { + // 0 args, read std in + // 1 arg, null input, process expression + // 1 arg, read file in sequence + // 2+ args, [0] = expression, file the rest + + var err error + stat, _ := os.Stdin.Stat() + pipingStdIn := (stat.Mode() & os.ModeCharDevice) == 0 + + out := cmd.OutOrStdout() + + fileInfo, _ := os.Stdout.Stat() + + if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) { + colorsEnabled = true + } + printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, printDocSeparators) + + switch len(args) { + case 0: + if pipingStdIn { + err = yqlib.EvaluateAllFileStreams("", []string{"-"}, printer) + } else { + cmd.Println(cmd.UsageString()) + return nil + } + case 1: + if nullInput { + err = yqlib.EvaluateAllFileStreams(args[0], []string{}, printer) + } else { + err = yqlib.EvaluateAllFileStreams("", []string{args[0]}, printer) + } + default: + err = yqlib.EvaluateAllFileStreams(args[0], args[1:len(args)], printer) + } + + cmd.SilenceUsage = true + return err +} diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go index fd41bf7..6307718 100644 --- a/cmd/evalute_sequence_command.go +++ b/cmd/evalute_sequence_command.go @@ -1,7 +1,6 @@ package cmd import ( - "container/list" "os" "github.com/mikefarah/yq/v4/pkg/yqlib" @@ -29,30 +28,10 @@ func evaluateSequence(cmd *cobra.Command, args []string) error { // 1 arg, read file in sequence // 2+ args, [0] = expression, file the rest - var matchingNodes *list.List var err error stat, _ := os.Stdin.Stat() pipingStdIn := (stat.Mode() & os.ModeCharDevice) == 0 - switch len(args) { - case 0: - if pipingStdIn { - matchingNodes, err = yqlib.Evaluate("-", "") - } else { - cmd.Println(cmd.UsageString()) - return nil - } - case 1: - if nullInput { - matchingNodes, err = yqlib.EvaluateExpression(args[0]) - } else { - matchingNodes, err = yqlib.Evaluate(args[0], "") - } - } - cmd.SilenceUsage = true - if err != nil { - return err - } out := cmd.OutOrStdout() fileInfo, _ := os.Stdout.Stat() @@ -60,7 +39,26 @@ func evaluateSequence(cmd *cobra.Command, args []string) error { if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) { colorsEnabled = true } - printer := yqlib.NewPrinter(outputToJSON, unwrapScalar, colorsEnabled, indent, printDocSeparators) + printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, printDocSeparators) - return printer.PrintResults(matchingNodes, out) + switch len(args) { + case 0: + if pipingStdIn { + err = yqlib.EvaluateFileStreamsSequence("", []string{"-"}, printer) + } else { + cmd.Println(cmd.UsageString()) + return nil + } + case 1: + if nullInput { + err = yqlib.EvaluateAllFileStreams(args[0], []string{}, printer) + } else { + err = yqlib.EvaluateFileStreamsSequence("", []string{args[0]}, printer) + } + default: + err = yqlib.EvaluateFileStreamsSequence(args[0], args[1:len(args)], printer) + } + + cmd.SilenceUsage = true + return err } diff --git a/cmd/root.go b/cmd/root.go index cae90a5..cdda014 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -65,6 +65,6 @@ func New() *cobra.Command { rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors") rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors") - rootCmd.AddCommand(createEvaluateSequenceCommand()) + rootCmd.AddCommand(createEvaluateSequenceCommand(), createEvaluateAllCommand()) return rootCmd } diff --git a/pkg/yqlib/document_index_operator_test.go b/pkg/yqlib/document_index_operator_test.go index 2a4a18b..9dd61dd 100644 --- a/pkg/yqlib/document_index_operator_test.go +++ b/pkg/yqlib/document_index_operator_test.go @@ -25,10 +25,10 @@ var documentIndexScenarios = []expressionScenario{ { description: "Print Document Index with matches", document: "a: cat\n---\na: frog\n", - expression: `.a | {"match": ., "doc": (. | documentIndex)}`, + expression: `.a | ({"match": ., "doc": (. | documentIndex)})`, expected: []string{ "D0, P[], (!!map)::match: cat\ndoc: 0\n", - "D1, P[], (!!map)::match: frog\ndoc: 1\n", + "D0, P[], (!!map)::match: frog\ndoc: 1\n", }, }, } diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 044e564..56030da 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -62,6 +62,7 @@ var DocumentFilter = &OperationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precede var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: SelfOperator} var ValueOp = &OperationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: ValueOperator} var Not = &OperationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: NotOperator} +var Empty = &OperationType{Type: "EMPTY", NumArgs: 50, Handler: EmptyOperator} var RecursiveDescent = &OperationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: RecursiveDescentOperator} diff --git a/pkg/yqlib/operator_collect_object.go b/pkg/yqlib/operator_collect_object.go index 4845aee..8cd48e8 100644 --- a/pkg/yqlib/operator_collect_object.go +++ b/pkg/yqlib/operator_collect_object.go @@ -17,8 +17,44 @@ import ( func CollectObjectOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) { log.Debugf("-- collectObjectOperation") - return collect(d, list.New(), matchMap) + if matchMap.Len() == 0 { + return list.New(), nil + } + first := matchMap.Front().Value.(*CandidateNode) + var rotated []*list.List = make([]*list.List, len(first.Node.Content)) + + for i := 0; i < len(first.Node.Content); i++ { + rotated[i] = list.New() + } + + for el := matchMap.Front(); el != nil; el = el.Next() { + candidateNode := el.Value.(*CandidateNode) + for i := 0; i < len(first.Node.Content); i++ { + rotated[i].PushBack(createChildCandidate(candidateNode, i)) + } + } + + newObject := list.New() + for i := 0; i < len(first.Node.Content); i++ { + additions, err := collect(d, list.New(), rotated[i]) + if err != nil { + return nil, err + } + newObject.PushBackList(additions) + } + + return newObject, nil + +} + +func createChildCandidate(candidate *CandidateNode, index int) *CandidateNode { + return &CandidateNode{ + Document: candidate.Document, + Path: append(candidate.Path, index), + Filename: candidate.Filename, + Node: candidate.Node.Content[index], + } } func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.List) (*list.List, error) { diff --git a/pkg/yqlib/operator_collect_object_test.go b/pkg/yqlib/operator_collect_object_test.go index 22c9957..58167dc 100644 --- a/pkg/yqlib/operator_collect_object_test.go +++ b/pkg/yqlib/operator_collect_object_test.go @@ -5,6 +5,26 @@ import ( ) var collectObjectOperatorScenarios = []expressionScenario{ + { + document: ``, + expression: `{}`, + expected: []string{}, + }, + { + document: "{name: Mike}\n", + expression: `{"wrap": .}`, + expected: []string{ + "D0, P[], (!!map)::wrap: {name: Mike}\n", + }, + }, + { + document: "{name: Mike}\n---\n{name: Bob}", + expression: `{"wrap": .}`, + expected: []string{ + "D0, P[], (!!map)::wrap: {name: Mike}\n", + "D0, P[], (!!map)::wrap: {name: Bob}\n", + }, + }, { document: `{name: Mike, age: 32}`, expression: `{.name: .age}`, @@ -20,6 +40,16 @@ var collectObjectOperatorScenarios = []expressionScenario{ "D0, P[], (!!map)::Mike: dog\n", }, }, + { + document: "{name: Mike, pets: [cat, dog]}\n---\n{name: Rosey, pets: [monkey, sheep]}", + expression: `{.name: .pets[]}`, + expected: []string{ + "D0, P[], (!!map)::Mike: cat\n", + "D0, P[], (!!map)::Mike: dog\n", + "D0, P[], (!!map)::Rosey: monkey\n", + "D0, P[], (!!map)::Rosey: sheep\n", + }, + }, { document: `{name: Mike, pets: [cat, dog], food: [hotdog, burger]}`, expression: `{.name: .pets[], "f":.food[]}`, @@ -55,11 +85,9 @@ b: {cows: [apl, bba]} }, { document: `{name: Mike}`, - expression: `{"wrap": {"further": .}}`, + expression: `{"wrap": {"further": .}} | (.. style= "flow")`, expected: []string{ - `D0, P[], (!!map)::wrap: - further: {name: Mike} -`, + "D0, P[], (!!map)::{wrap: {further: {name: Mike}}}\n", }, }, } @@ -68,4 +96,5 @@ func TestCollectObjectOperatorScenarios(t *testing.T) { for _, tt := range collectObjectOperatorScenarios { testScenario(t, &tt) } + documentScenarios(t, "Collect into Object", collectObjectOperatorScenarios) } diff --git a/pkg/yqlib/operator_collect_test.go b/pkg/yqlib/operator_collect_test.go index a601818..0f2501f 100644 --- a/pkg/yqlib/operator_collect_test.go +++ b/pkg/yqlib/operator_collect_test.go @@ -6,25 +6,30 @@ import ( var collectOperatorScenarios = []expressionScenario{ { - document: `{}`, + document: ``, + expression: `[]`, + expected: []string{}, + }, + { + document: ``, expression: `["cat"]`, expected: []string{ "D0, P[], (!!seq)::- cat\n", }, }, { - document: `{}`, + document: ``, expression: `[true]`, expected: []string{ "D0, P[], (!!seq)::- true\n", }, }, { - document: `{}`, + document: ``, expression: `["cat", "dog"]`, expected: []string{ "D0, P[], (!!seq)::- cat\n- dog\n", }, }, { - document: `{}`, + document: ``, expression: `1 | collect`, expected: []string{ "D0, P[], (!!seq)::- 1\n", @@ -48,4 +53,5 @@ func TestCollectOperatorScenarios(t *testing.T) { for _, tt := range collectOperatorScenarios { testScenario(t, &tt) } + documentScenarios(t, "Collect into Array", collectOperatorScenarios) } diff --git a/pkg/yqlib/operator_create_map.go b/pkg/yqlib/operator_create_map.go index 6c4d994..28cbad9 100644 --- a/pkg/yqlib/operator_create_map.go +++ b/pkg/yqlib/operator_create_map.go @@ -8,15 +8,50 @@ import ( func CreateMapOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { log.Debugf("-- createMapOperation") - var path []interface{} = nil + + //each matchingNodes entry should turn into a sequence of keys to create. + //then collect object should do a cross function of the same index sequence for all matches. + + var path []interface{} + var document uint = 0 - if matchingNodes.Front() != nil { - sample := matchingNodes.Front().Value.(*CandidateNode) - path = sample.Path - document = sample.Document + + sequences := list.New() + + if matchingNodes.Len() > 0 { + + for matchingNodeEl := matchingNodes.Front(); matchingNodeEl != nil; matchingNodeEl = matchingNodeEl.Next() { + matchingNode := matchingNodeEl.Value.(*CandidateNode) + sequenceNode, err := sequenceFor(d, matchingNode, pathNode) + if err != nil { + return nil, err + } + sequences.PushBack(sequenceNode) + } + } else { + sequenceNode, err := sequenceFor(d, nil, pathNode) + if err != nil { + return nil, err + } + sequences.PushBack(sequenceNode) } - mapPairs, err := crossFunction(d, matchingNodes, pathNode, + return nodeToMap(&CandidateNode{Node: listToNodeSeq(sequences), Document: document, Path: path}), nil + +} + +func sequenceFor(d *dataTreeNavigator, matchingNode *CandidateNode, pathNode *PathTreeNode) (*CandidateNode, error) { + var path []interface{} + var document uint = 0 + var matches = list.New() + + if matchingNode != nil { + path = matchingNode.Path + document = matchingNode.Document + matches = nodeToMap(matchingNode) + } + + mapPairs, err := crossFunction(d, matches, pathNode, func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { node := yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} log.Debugf("LHS:", NodeToString(lhs)) @@ -32,12 +67,19 @@ func CreateMapOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode if err != nil { return nil, err } - //wrap up all the pairs into an array - node := yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - for mapPair := mapPairs.Front(); mapPair != nil; mapPair = mapPair.Next() { - mapPairCandidate := mapPair.Value.(*CandidateNode) - log.Debugf("Collecting %v into sequence", NodeToString(mapPairCandidate)) - node.Content = append(node.Content, mapPairCandidate.Node) - } - return nodeToMap(&CandidateNode{Node: &node, Document: document, Path: path}), nil + innerList := listToNodeSeq(mapPairs) + innerList.Style = yaml.FlowStyle + return &CandidateNode{Node: innerList, Document: document, Path: path}, nil +} + +//NOTE: here the document index gets dropped so we +// no longer know where the node originates from. +func listToNodeSeq(list *list.List) *yaml.Node { + node := yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + for entry := list.Front(); entry != nil; entry = entry.Next() { + entryCandidate := entry.Value.(*CandidateNode) + log.Debugf("Collecting %v into sequence", NodeToString(entryCandidate)) + node.Content = append(node.Content, entryCandidate.Node) + } + return &node } diff --git a/pkg/yqlib/operator_create_map_test.go b/pkg/yqlib/operator_create_map_test.go index 24d25ad..291abd0 100644 --- a/pkg/yqlib/operator_create_map_test.go +++ b/pkg/yqlib/operator_create_map_test.go @@ -5,41 +5,71 @@ import ( ) var createMapOperatorScenarios = []expressionScenario{ + { + document: ``, + expression: `"frog": "jumps"`, + expected: []string{ + "D0, P[], (!!seq)::- [{frog: jumps}]\n", + }, + }, { document: `{name: Mike, age: 32}`, expression: `.name: .age`, expected: []string{ - "D0, P[], (!!seq)::- Mike: 32\n", + "D0, P[], (!!seq)::- [{Mike: 32}]\n", }, }, { document: `{name: Mike, pets: [cat, dog]}`, expression: `.name: .pets[]`, expected: []string{ - "D0, P[], (!!seq)::- Mike: cat\n- Mike: dog\n", + "D0, P[], (!!seq)::- [{Mike: cat}, {Mike: dog}]\n", }, }, { document: `{name: Mike, pets: [cat, dog], food: [hotdog, burger]}`, expression: `.name: .pets[], "f":.food[]`, expected: []string{ - "D0, P[], (!!seq)::- Mike: cat\n- Mike: dog\n", - "D0, P[], (!!seq)::- f: hotdog\n- f: burger\n", + "D0, P[], (!!seq)::- [{Mike: cat}, {Mike: dog}]\n", + "D0, P[], (!!seq)::- [{f: hotdog}, {f: burger}]\n", + }, + }, + { + document: "{name: Mike, pets: [cat, dog], food: [hotdog, burger]}\n---\n{name: Fred, pets: [mouse], food: [pizza, onion, apple]}", + expression: `.name: .pets[], "f":.food[]`, + expected: []string{ + "D0, P[], (!!seq)::- [{Mike: cat}, {Mike: dog}]\n- [{Fred: mouse}]\n", + "D0, P[], (!!seq)::- [{f: hotdog}, {f: burger}]\n- [{f: pizza}, {f: onion}, {f: apple}]\n", }, }, { document: `{name: Mike, pets: {cows: [apl, bba]}}`, expression: `"a":.name, "b":.pets`, expected: []string{ - "D0, P[], (!!seq)::- a: Mike\n", - "D0, P[], (!!seq)::- b: {cows: [apl, bba]}\n", + "D0, P[], (!!seq)::- [{a: Mike}]\n", + "D0, P[], (!!seq)::- [{b: {cows: [apl, bba]}}]\n", }, }, { document: `{name: Mike}`, expression: `"wrap": .`, expected: []string{ - "D0, P[], (!!seq)::- wrap: {name: Mike}\n", + "D0, P[], (!!seq)::- [{wrap: {name: Mike}}]\n", + }, + }, + { + document: "{name: Mike}\n---\n{name: Bob}", + expression: `"wrap": .`, + expected: []string{ + "D0, P[], (!!seq)::- [{wrap: {name: Mike}}]\n- [{wrap: {name: Bob}}]\n", + }, + }, + { + document: "{name: Mike}\n---\n{name: Bob}", + expression: `"wrap": ., .name: "great"`, + expected: []string{ + "D0, P[], (!!seq)::- [{wrap: {name: Mike}}]\n- [{wrap: {name: Bob}}]\n", + "D0, P[], (!!seq)::- [{Mike: great}]\n- [{Bob: great}]\n", }, }, } diff --git a/pkg/yqlib/operators.go b/pkg/yqlib/operators.go index b1dfc1b..f99bca1 100644 --- a/pkg/yqlib/operators.go +++ b/pkg/yqlib/operators.go @@ -16,6 +16,10 @@ func UnwrapDoc(node *yaml.Node) *yaml.Node { return node } +func EmptyOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { + return list.New(), nil +} + func PipeOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs) if err != nil { diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 7c271ad..350a3b6 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -23,11 +23,23 @@ type expressionScenario struct { func testScenario(t *testing.T, s *expressionScenario) { var results *list.List var err error - if s.document != "" { - results, err = EvaluateStream("sample.yaml", strings.NewReader(s.document), s.expression) - } else { - results, err = EvaluateExpression(s.expression) + + node, err := treeCreator.ParsePath(s.expression) + if err != nil { + t.Error(err) + return } + inputs := list.New() + + if s.document != "" { + inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml") + if err != nil { + t.Error(err) + return + } + } + + results, err = treeNavigator.GetMatchingNodes(inputs, node) if err != nil { t.Error(err) @@ -40,15 +52,13 @@ func documentScenarios(t *testing.T, title string, scenarios []expressionScenari f, err := os.Create(fmt.Sprintf("doc/%v.md", title)) if err != nil { - panic(err) + t.Error(err) } defer f.Close() w := bufio.NewWriter(f) w.WriteString(fmt.Sprintf("# %v\n", title)) w.WriteString(fmt.Sprintf("## Examples\n")) - printer := NewPrinter(false, true, false, 2, true) - for index, s := range scenarios { if !s.skipDoc { @@ -69,20 +79,23 @@ func documentScenarios(t *testing.T, title string, scenarios []expressionScenari w.WriteString(fmt.Sprintf("Result\n")) var output bytes.Buffer - var results *list.List var err error - if s.document != "" { - results, err = EvaluateStream("sample.yaml", strings.NewReader(s.document), s.expression) - } else { - results, err = EvaluateExpression(s.expression) - } + printer := NewPrinter(bufio.NewWriter(&output), false, true, false, 2, true) - printer.PrintResults(results, bufio.NewWriter(&output)) + if s.document != "" { + node, err := treeCreator.ParsePath(s.expression) + if err != nil { + t.Error(err) + } + err = EvaluateStream("sample.yaml", strings.NewReader(s.document), node, printer) + } else { + err = EvaluateAllFileStreams(s.expression, []string{}, printer) + } w.WriteString(fmt.Sprintf("```yaml\n%v```\n", output.String())) if err != nil { - panic(err) + t.Error(err) } } diff --git a/pkg/yqlib/path_parse_test.go b/pkg/yqlib/path_parse_test.go index cc2e2e3..45037f3 100644 --- a/pkg/yqlib/path_parse_test.go +++ b/pkg/yqlib/path_parse_test.go @@ -114,6 +114,11 @@ var pathTests = []struct { append(make([]interface{}, 0), "foo*", "PIPE", "(", "SELF", "ASSIGN_STYLE", "flow (string)", ")"), append(make([]interface{}, 0), "foo*", "SELF", "flow (string)", "ASSIGN_STYLE", "PIPE"), }, + { + `{}`, + append(make([]interface{}, 0), "{", "}"), + append(make([]interface{}, 0), "EMPTY", "COLLECT_OBJECT", "PIPE"), + }, // {".animals | .==cat", append(make([]interface{}, 0), "animals", "TRAVERSE", "SELF", "EQUALS", "cat")}, // {".animals | (. == cat)", append(make([]interface{}, 0), "animals", "TRAVERSE", "(", "SELF", "EQUALS", "cat", ")")}, diff --git a/pkg/yqlib/path_postfix.go b/pkg/yqlib/path_postfix.go index 992a48e..1f360e4 100644 --- a/pkg/yqlib/path_postfix.go +++ b/pkg/yqlib/path_postfix.go @@ -41,8 +41,14 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, er opener = OpenCollectObject collectOperator = CollectObject } + itemsInMiddle := false for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != opener { opStack, result = popOpToResult(opStack, result) + itemsInMiddle = true + } + if !itemsInMiddle { + // must be an empty collection, add the empty object as a LHS parameter + result = append(result, &Operation{OperationType: Empty}) } if len(opStack) == 0 { return nil, errors.New("Bad path expression, got close collect brackets without matching opening bracket") diff --git a/pkg/yqlib/path_tokeniser.go b/pkg/yqlib/path_tokeniser.go index a952ff6..e179ab7 100644 --- a/pkg/yqlib/path_tokeniser.go +++ b/pkg/yqlib/path_tokeniser.go @@ -174,7 +174,6 @@ func selfToken() lex.Action { } } -// Creates the lexer object and compiles the NFA. func initLexer() (*lex.Lexer, error) { lexer := lex.NewLexer() lexer.Add([]byte(`\(`), literalToken(OpenBracket, false)) @@ -206,9 +205,7 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`footComment`), opTokenWithPrefs(GetComment, &CommentOpPreferences{FootComment: true})) lexer.Add([]byte(`comments\s*=`), opTokenWithPrefs(AssignComment, &CommentOpPreferences{LineComment: true, HeadComment: true, FootComment: true})) - // lexer.Add([]byte(`style`), opToken(GetStyle)) - // lexer.Add([]byte(`and`), opToken()) lexer.Add([]byte(`collect`), opToken(Collect)) lexer.Add([]byte(`\s*==\s*`), opToken(Equals)) @@ -222,8 +219,7 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte("( |\t|\n|\r)+"), skip) - lexer.Add([]byte(`d[0-9]+`), documentToken()) // $0 - + lexer.Add([]byte(`d[0-9]+`), documentToken()) lexer.Add([]byte(`\."[^ "]+"`), pathToken(true)) lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+`), pathToken(false)) lexer.Add([]byte(`\.`), selfToken()) @@ -248,7 +244,6 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`\}`), literalToken(CloseCollectObject, true)) lexer.Add([]byte(`\*`), opToken(Multiply)) - // lexer.Add([]byte(`[^ \,\|\.\[\(\)=]+`), stringValue(false)) err := lexer.Compile() if err != nil { return nil, err diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index 280669b..eb5d8a8 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -9,7 +9,7 @@ import ( ) type Printer interface { - PrintResults(matchingNodes *list.List, writer io.Writer) error + PrintResults(matchingNodes *list.List) error } type resultsPrinter struct { @@ -18,10 +18,20 @@ type resultsPrinter struct { colorsEnabled bool indent int printDocSeparators bool + writer io.Writer + firstTimePrinting bool } -func NewPrinter(outputToJSON bool, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer { - return &resultsPrinter{outputToJSON, unwrapScalar, colorsEnabled, indent, printDocSeparators} +func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer { + return &resultsPrinter{ + writer: writer, + outputToJSON: outputToJSON, + unwrapScalar: unwrapScalar, + colorsEnabled: colorsEnabled, + indent: indent, + printDocSeparators: printDocSeparators, + firstTimePrinting: true, + } } func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error { @@ -42,7 +52,7 @@ func (p *resultsPrinter) writeString(writer io.Writer, txt string) error { return errorWriting } -func (p *resultsPrinter) PrintResults(matchingNodes *list.List, writer io.Writer) error { +func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { var err error if p.outputToJSON { explodeOp := Operation{OperationType: Explode} @@ -53,7 +63,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List, writer io.Writer } } - bufferedWriter := bufio.NewWriter(writer) + bufferedWriter := bufio.NewWriter(p.writer) defer safelyFlush(bufferedWriter) if matchingNodes.Len() == 0 { @@ -61,12 +71,12 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List, writer io.Writer return nil } - var previousDocIndex uint = matchingNodes.Front().Value.(*CandidateNode).Document + previousDocIndex := matchingNodes.Front().Value.(*CandidateNode).Document for el := matchingNodes.Front(); el != nil; el = el.Next() { mappedDoc := el.Value.(*CandidateNode) - if previousDocIndex != mappedDoc.Document && p.printDocSeparators { + if (!p.firstTimePrinting || (previousDocIndex != mappedDoc.Document)) && p.printDocSeparators { p.writeString(bufferedWriter, "---\n") } @@ -76,6 +86,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List, writer io.Writer previousDocIndex = mappedDoc.Document } + p.firstTimePrinting = false return nil } diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go index af104ef..83dcf82 100644 --- a/pkg/yqlib/utils.go +++ b/pkg/yqlib/utils.go @@ -13,35 +13,14 @@ var treeNavigator = NewDataTreeNavigator(NavigationPrefs{}) var treeCreator = NewPathTreeCreator() func readStream(filename string) (io.Reader, error) { - var stream io.Reader if filename == "-" { - stream = bufio.NewReader(os.Stdin) + return bufio.NewReader(os.Stdin), nil } else { - file, err := os.Open(filename) // nolint gosec - if err != nil { - return nil, err - } - defer safelyCloseFile(file) - stream = file + return os.Open(filename) // nolint gosec } - return stream, nil } -func EvaluateExpression(expression string) (*list.List, error) { - node, err := treeCreator.ParsePath(expression) - if err != nil { - return nil, err - } - return treeNavigator.GetMatchingNodes(list.New(), node) -} - -func EvaluateStream(filename string, reader io.Reader, expression string) (*list.List, error) { - node, err := treeCreator.ParsePath(expression) - if err != nil { - return nil, err - } - - var matchingNodes = list.New() +func EvaluateStream(filename string, reader io.Reader, node *PathTreeNode, printer Printer) error { var currentIndex uint = 0 @@ -51,9 +30,9 @@ func EvaluateStream(filename string, reader io.Reader, expression string) (*list errorReading := decoder.Decode(&dataBucket) if errorReading == io.EOF { - return matchingNodes, nil + return nil } else if errorReading != nil { - return nil, errorReading + return errorReading } candidateNode := &CandidateNode{ Document: currentIndex, @@ -63,23 +42,92 @@ func EvaluateStream(filename string, reader io.Reader, expression string) (*list inputList := list.New() inputList.PushBack(candidateNode) - newMatches, errorParsing := treeNavigator.GetMatchingNodes(inputList, node) + matches, errorParsing := treeNavigator.GetMatchingNodes(inputList, node) if errorParsing != nil { - return nil, errorParsing + return errorParsing } - matchingNodes.PushBackList(newMatches) + printer.PrintResults(matches) currentIndex = currentIndex + 1 } } -func Evaluate(filename string, expression string) (*list.List, error) { +func readDocuments(reader io.Reader, filename string) (*list.List, error) { + decoder := yaml.NewDecoder(reader) + inputList := list.New() + var currentIndex uint = 0 - var reader, err = readStream(filename) - if err != nil { - return nil, err + for { + var dataBucket yaml.Node + errorReading := decoder.Decode(&dataBucket) + + if errorReading == io.EOF { + switch reader.(type) { + case *os.File: + safelyCloseFile(reader.(*os.File)) + } + return inputList, nil + } else if errorReading != nil { + return nil, errorReading + } + candidateNode := &CandidateNode{ + Document: currentIndex, + Filename: filename, + Node: &dataBucket, + } + + inputList.PushBack(candidateNode) + + currentIndex = currentIndex + 1 } - return EvaluateStream(filename, reader, expression) +} +func EvaluateAllFileStreams(expression string, filenames []string, printer Printer) error { + node, err := treeCreator.ParsePath(expression) + if err != nil { + return err + } + var allDocuments *list.List = list.New() + for _, filename := range filenames { + reader, err := readStream(filename) + if err != nil { + return err + } + fileDocuments, err := readDocuments(reader, filename) + if err != nil { + return err + } + allDocuments.PushBackList(fileDocuments) + } + matches, err := treeNavigator.GetMatchingNodes(allDocuments, node) + if err != nil { + return err + } + return printer.PrintResults(matches) +} + +func EvaluateFileStreamsSequence(expression string, filenames []string, printer Printer) error { + + node, err := treeCreator.ParsePath(expression) + if err != nil { + return err + } + + for _, filename := range filenames { + reader, err := readStream(filename) + if err != nil { + return err + } + err = EvaluateStream(filename, reader, node, printer) + if err != nil { + return err + } + + switch reader.(type) { + case *os.File: + safelyCloseFile(reader.(*os.File)) + } + } + return nil } func safelyRenameFile(from string, to string) {