diff --git a/go.mod b/go.mod index 64cafbf..4152391 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ require ( github.com/elliotchance/orderedmap v1.3.0 // indirect github.com/fatih/color v1.9.0 github.com/goccy/go-yaml v1.8.1 + github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.7 // indirect github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index c12b43c..b36faa9 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a h1:zPPuIq2jAWWPTrGt70eK/BSch+gFAGrNzecsoENgu2o= +github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= diff --git a/pkg/yqlib/treeops/candidate_node.go b/pkg/yqlib/treeops/candidate_node.go index e714fc1..7c3b3f9 100644 --- a/pkg/yqlib/treeops/candidate_node.go +++ b/pkg/yqlib/treeops/candidate_node.go @@ -5,6 +5,7 @@ import ( "strconv" "strings" + "github.com/jinzhu/copier" "gopkg.in/yaml.v3" ) @@ -18,6 +19,12 @@ func (n *CandidateNode) GetKey() string { return fmt.Sprintf("%v - %v - %v", n.Document, n.Path, n.Node.Value) } +func (n *CandidateNode) Copy() *CandidateNode { + clone := &CandidateNode{} + copier.Copy(clone, n) + return clone +} + // updates this candidate from the given candidate node func (n *CandidateNode) UpdateFrom(other *CandidateNode) { n.UpdateAttributesFrom(other) @@ -34,7 +41,10 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) { } n.Node.Kind = other.Node.Kind n.Node.Tag = other.Node.Tag - n.Node.Style = other.Node.Style + // not sure if this ever should happen here... + // if other.Node.Style != 0 { + // n.Node.Style = other.Node.Style + // } n.Node.FootComment = other.Node.FootComment n.Node.HeadComment = other.Node.HeadComment n.Node.LineComment = other.Node.LineComment diff --git a/pkg/yqlib/treeops/operation_collection_object_test.go b/pkg/yqlib/treeops/operation_collection_object_test.go index 4a522df..62333ee 100644 --- a/pkg/yqlib/treeops/operation_collection_object_test.go +++ b/pkg/yqlib/treeops/operation_collection_object_test.go @@ -1,2 +1,39 @@ package treeops +import ( + "testing" +) + +var collectObjectOperatorScenarios = []expressionScenario{ + { + document: `{name: Mike, age: 32}`, + expression: `{.name: .age}`, + expected: []string{ + "D0, P[0], (!!map)::Mike: 32\n", + }, + }, + { + document: `{name: Mike, pets: [cat, dog]}`, + expression: `{.name: .pets[]}`, + expected: []string{ + "D0, P[0], (!!map)::Mike: cat\n", + "D0, P[1], (!!map)::Mike: dog\n", + }, + }, + { + document: `{name: Mike, pets: [cat, dog], food: [hotdog, burger]}`, + expression: `{.name: .pets[], "f":.food[]}`, + expected: []string{ + "D0, P[], (!!map)::Mike: cat\nf: hotdog\n", + "D0, P[], (!!map)::Mike: cat\nf: burger\n", + "D0, P[], (!!map)::Mike: dog\nf: hotdog\n", + "D0, P[], (!!map)::Mike: dog\nf: burger\n", + }, + }, +} + +func TestCollectObjectOperatorScenarios(t *testing.T) { + for _, tt := range collectObjectOperatorScenarios { + testScenario(t, &tt) + } +} diff --git a/pkg/yqlib/treeops/operator_collect_object.go b/pkg/yqlib/treeops/operator_collect_object.go index 74ca176..4cfee96 100644 --- a/pkg/yqlib/treeops/operator_collect_object.go +++ b/pkg/yqlib/treeops/operator_collect_object.go @@ -1,8 +1,57 @@ package treeops -import "container/list" +import ( + "container/list" +) + +/* +[Mike: cat, Bob: dog] +[Thing: rabbit, peter: sam] + +==> cross multiply + +{Mike: cat, Thing: rabbit} +{Mike: cat, peter: sam} +... +*/ func CollectObjectOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) { log.Debugf("-- collectObjectOperation") - return nil, nil + return collect(d, list.New(), matchMap) + +} + +func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.List) (*list.List, error) { + if remainingMatches.Len() == 0 { + return aggregate, nil + } + + candidate := remainingMatches.Remove(remainingMatches.Front()).(*CandidateNode) + splatted, err := Splat(d, nodeToMap(candidate)) + if err != nil { + return nil, err + } + + if aggregate.Len() == 0 { + return collect(d, splatted, remainingMatches) + } + + newAgg := list.New() + + for el := aggregate.Front(); el != nil; el = el.Next() { + aggCandidate := el.Value.(*CandidateNode) + for splatEl := splatted.Front(); splatEl != nil; splatEl = splatEl.Next() { + splatCandidate := splatEl.Value.(*CandidateNode) + newCandidate := aggCandidate.Copy() + newCandidate.Path = nil + + newCandidate, err := multiply(d, newCandidate, splatCandidate) + if err != nil { + return nil, err + } + newAgg.PushBack(newCandidate) + } + } + return collect(d, newAgg, remainingMatches) + } diff --git a/pkg/yqlib/treeops/operator_create_map_test.go b/pkg/yqlib/treeops/operator_create_map_test.go index 39a2e6e..af0bd22 100644 --- a/pkg/yqlib/treeops/operator_create_map_test.go +++ b/pkg/yqlib/treeops/operator_create_map_test.go @@ -19,6 +19,14 @@ var createMapOperatorScenarios = []expressionScenario{ "D0, P[], (!!seq)::- Mike: cat\n- Mike: dog\n", }, }, + { + document: `{name: Mike, pets: [cat, dog], food: [hotdog, burger]}`, + expression: `.name: .pets[], "f":.food[]`, + expected: []string{ + "D0, P[], (!!seq)::- Mike: cat\n- Mike: dog\n", + "D0, P[], (!!seq)::- f: hotdog\n- f: burger\n", + }, + }, } func TestCreateMapOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/treeops/operator_multiply_test.go b/pkg/yqlib/treeops/operator_multiply_test.go index 525ddd0..943d9fd 100644 --- a/pkg/yqlib/treeops/operator_multiply_test.go +++ b/pkg/yqlib/treeops/operator_multiply_test.go @@ -6,67 +6,65 @@ import ( var multiplyOperatorScenarios = []expressionScenario{ { - // document: `{a: {also: [1]}, b: {also: me}}`, - // expression: `.a * .b`, - // expected: []string{ - // "D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n", - // }, - // }, { - // document: `{a: {also: me}, b: {also: [1]}}`, - // expression: `.a * .b`, - // expected: []string{ - // "D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\n", - // }, - // }, { - // document: `{a: {also: me}, b: {also: {g: wizz}}}`, - // expression: `.a * .b`, - // expected: []string{ - // "D0, P[], (!!map)::{a: {also: {g: wizz}}, b: {also: {g: wizz}}}\n", - // }, - // }, { - // document: `{a: {also: {g: wizz}}, b: {also: me}}`, - // expression: `.a * .b`, - // expected: []string{ - // "D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n", - // }, - // }, { - // document: `{a: {also: {g: wizz}}, b: {also: [1]}}`, - // expression: `.a * .b`, - // expected: []string{ - // "D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\n", - // }, - // }, { - // document: `{a: {also: [1]}, b: {also: {g: wizz}}}`, - // expression: `.a * .b`, - // expected: []string{ - // "D0, P[], (!!map)::{a: {also: {g: wizz}}, b: {also: {g: wizz}}}\n", - // }, - // }, { - // document: `{a: {things: great}, b: {also: me}}`, - // expression: `.a * .b`, - // expected: []string{ - // "D0, P[], (!!map)::{a: {things: great, also: me}, b: {also: me}}\n", - // }, - // }, { - // document: `a: {things: great} - // b: - // also: "me" - // `, - // expression: `(.a * .b)`, - // expected: []string{ - // `D0, P[], (!!map)::a: - // things: great - // also: "me" - // b: - // also: "me" - // `, - // }, - // }, { - // document: `{a: [1,2,3], b: [3,4,5]}`, - // expression: `.a * .b`, - // expected: []string{ - // "D0, P[], (!!map)::{a: [3, 4, 5], b: [3, 4, 5]}\n", - // }, + document: `{a: {also: [1]}, b: {also: me}}`, + expression: `. * {"a" : .b}`, + expected: []string{ + "D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n", + }, + }, { + document: `{a: {also: me}, b: {also: [1]}}`, + expression: `. * {"a":.b}`, + expected: []string{ + "D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\n", + }, + }, { + document: `{a: {also: me}, b: {also: {g: wizz}}}`, + expression: `. * {"a":.b}`, + expected: []string{ + "D0, P[], (!!map)::{a: {also: {g: wizz}}, b: {also: {g: wizz}}}\n", + }, + }, { + document: `{a: {also: {g: wizz}}, b: {also: me}}`, + expression: `. * {"a":.b}`, + expected: []string{ + "D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n", + }, + }, { + document: `{a: {also: {g: wizz}}, b: {also: [1]}}`, + expression: `. * {"a":.b}`, + expected: []string{ + "D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\n", + }, + }, { + document: `{a: {also: [1]}, b: {also: {g: wizz}}}`, + expression: `. * {"a":.b}`, + expected: []string{ + "D0, P[], (!!map)::{a: {also: {g: wizz}}, b: {also: {g: wizz}}}\n", + }, + }, { + document: `{a: {things: great}, b: {also: me}}`, + expression: `. * {"a":.b}`, + expected: []string{ + "D0, P[], (!!map)::{a: {things: great, also: me}, b: {also: me}}\n", + }, + }, { + document: `a: {things: great} +b: + also: "me" +`, + expression: `. * {"a":.b}`, + expected: []string{ + `D0, P[], (!!map)::a: {things: great, also: me} +b: + also: "me" +`, + }, + }, { + document: `{a: [1,2,3], b: [3,4,5]}`, + expression: `. * {"a":.b}`, + expected: []string{ + "D0, P[], (!!map)::{a: [3, 4, 5], b: [3, 4, 5]}\n", + }, }, } diff --git a/pkg/yqlib/treeops/operator_recursive_descent.go b/pkg/yqlib/treeops/operator_recursive_descent.go index 3609846..7d7511b 100644 --- a/pkg/yqlib/treeops/operator_recursive_descent.go +++ b/pkg/yqlib/treeops/operator_recursive_descent.go @@ -16,14 +16,11 @@ func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNod } func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List) error { - splatOperation := &Operation{OperationType: TraversePath, Value: "[]"} - splatTreeNode := &PathTreeNode{Operation: splatOperation} - for el := matchMap.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) results.PushBack(candidate) - children, err := TraversePathOperator(d, nodeToMap(candidate), splatTreeNode) + children, err := Splat(d, nodeToMap(candidate)) if err != nil { return err diff --git a/pkg/yqlib/treeops/operator_traverse_path.go b/pkg/yqlib/treeops/operator_traverse_path.go index dc05c34..2dba41d 100644 --- a/pkg/yqlib/treeops/operator_traverse_path.go +++ b/pkg/yqlib/treeops/operator_traverse_path.go @@ -8,6 +8,12 @@ import ( "gopkg.in/yaml.v3" ) +func Splat(d *dataTreeNavigator, matches *list.List) (*list.List, error) { + splatOperation := &Operation{OperationType: TraversePath, Value: "[]"} + splatTreeNode := &PathTreeNode{Operation: splatOperation} + return TraversePathOperator(d, matches, splatTreeNode) +} + func TraversePathOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) { log.Debugf("-- Traversing") var matchingNodeMap = list.New() diff --git a/pkg/yqlib/treeops/path_parse_test.go b/pkg/yqlib/treeops/path_parse_test.go index 825b42a..d5598c8 100644 --- a/pkg/yqlib/treeops/path_parse_test.go +++ b/pkg/yqlib/treeops/path_parse_test.go @@ -59,6 +59,11 @@ var pathTests = []struct { append(make([]interface{}, 0), "[", "true (bool)", "]"), append(make([]interface{}, 0), "true (bool)", "COLLECT", "PIPE"), }, + { + `[true, false]`, + append(make([]interface{}, 0), "[", "true (bool)", "UNION", "false (bool)", "]"), + append(make([]interface{}, 0), "true (bool)", "false (bool)", "UNION", "COLLECT", "PIPE"), + }, { `"mike": .a`, append(make([]interface{}, 0), "mike (string)", "CREATE_MAP", "a"), @@ -69,6 +74,21 @@ var pathTests = []struct { append(make([]interface{}, 0), "a", "CREATE_MAP", "mike (string)"), append(make([]interface{}, 0), "a", "mike (string)", "CREATE_MAP"), }, + { + `{"mike": .a}`, + append(make([]interface{}, 0), "{", "mike (string)", "CREATE_MAP", "a", "}"), + append(make([]interface{}, 0), "mike (string)", "a", "CREATE_MAP", "COLLECT_OBJECT", "PIPE"), + }, + { + `{.a: "mike"}`, + append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "mike (string)", "}"), + append(make([]interface{}, 0), "a", "mike (string)", "CREATE_MAP", "COLLECT_OBJECT", "PIPE"), + }, + { + `{.a: .c, .b[]: .f.g[]}`, + append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "PIPE", "[]", "CREATE_MAP", "f", "PIPE", "g", "PIPE", "[]", "}"), + append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "[]", "PIPE", "f", "g", "PIPE", "[]", "PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "PIPE"), + }, // {".animals | .==cat", append(make([]interface{}, 0), "animals", "TRAVERSE", "SELF", "EQUALS", "cat")}, // {".animals | (. == cat)", append(make([]interface{}, 0), "animals", "TRAVERSE", "(", "SELF", "EQUALS", "cat", ")")}, diff --git a/pkg/yqlib/treeops/path_postfix.go b/pkg/yqlib/treeops/path_postfix.go index 850307d..147e74d 100644 --- a/pkg/yqlib/treeops/path_postfix.go +++ b/pkg/yqlib/treeops/path_postfix.go @@ -32,10 +32,16 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, er for _, token := range tokens { log.Debugf("postfix processing token %v, %v", token.toString(), token.Operation) switch token.TokenType { - case OpenBracket, OpenCollect: + case OpenBracket, OpenCollect, OpenCollectObject: opStack = append(opStack, token) - case CloseCollect: - for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != OpenCollect { + case CloseCollect, CloseCollectObject: + var opener TokenType = OpenCollect + var collectOperator *OperationType = Collect + if token.TokenType == CloseCollectObject { + opener = OpenCollectObject + collectOperator = CollectObject + } + for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != opener { opStack, result = popOpToResult(opStack, result) } if len(opStack) == 0 { @@ -45,7 +51,7 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, er opStack = opStack[0 : len(opStack)-1] //and append a collect to the opStack opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: Pipe}}) - opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: Collect}}) + opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: collectOperator}}) case CloseBracket: for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != OpenBracket { opStack, result = popOpToResult(opStack, result)