Fixed numeric map key issue

2025-01-18 04:53:17 -08:00 · 2020-02-12 15:40:21 +11:00
parent f084f2bb23
commit 24e906bae6
9 changed files with 237 additions and 119 deletions
--- a/pkg/yqlib/data_navigator.go
+++ b/pkg/yqlib/data_navigator.go
@@ -1,13 +1,14 @@
 package yqlib

 import (
+	"fmt"
 	"strconv"

 	yaml "gopkg.in/yaml.v3"
 )

 type DataNavigator interface {
-	Traverse(value *yaml.Node, path []string) error
+	Traverse(value *yaml.Node, path []interface{}) error
 }

 type navigator struct {
@@ -20,7 +21,7 @@ func NewDataNavigator(NavigationStrategy NavigationStrategy) DataNavigator {
 	}
 }

-func (n *navigator) Traverse(value *yaml.Node, path []string) error {
+func (n *navigator) Traverse(value *yaml.Node, path []interface{}) error {
 	realValue := value
 	emptyArray := make([]interface{}, 0)
 	if realValue.Kind == yaml.DocumentNode {
@@ -30,7 +31,7 @@ func (n *navigator) Traverse(value *yaml.Node, path []string) error {
 	return n.doTraverse(value, "", path, emptyArray)
 }

-func (n *navigator) doTraverse(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
+func (n *navigator) doTraverse(value *yaml.Node, head interface{}, tail []interface{}, pathStack []interface{}) error {

 	log.Debug("head %v", head)
 	DebugNode(value)
@@ -65,23 +66,27 @@ func (n *navigator) getOrReplace(original *yaml.Node, expectedKind yaml.Kind) *y
 	return original
 }

-func (n *navigator) recurse(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
+func (n *navigator) recurse(value *yaml.Node, head interface{}, tail []interface{}, pathStack []interface{}) error {
 	log.Debug("recursing, processing %v, pathStack %v", head, pathStackToString(pathStack))
 	switch value.Kind {
 	case yaml.MappingNode:
 		log.Debug("its a map with %v entries", len(value.Content)/2)
-		return n.recurseMap(value, head, tail, pathStack)
+		headString := fmt.Sprintf("%v", head)
+		return n.recurseMap(value, headString, tail, pathStack)
+
 	case yaml.SequenceNode:
 		log.Debug("its a sequence of %v things!", len(value.Content))

-		var index, errorParsingIndex = strconv.ParseInt(head, 10, 64) // nolint
-		if errorParsingIndex == nil {
-			return n.recurseArray(value, index, head, tail, pathStack)
-		} else if head == "+" {
-			return n.appendArray(value, head, tail, pathStack)
-		}
-		return n.splatArray(value, head, tail, pathStack)
+		switch head := head.(type) {
+		case int64:
+			return n.recurseArray(value, head, head, tail, pathStack)
+		default:

+			if head == "+" {
+				return n.appendArray(value, head, tail, pathStack)
+			}
+			return n.splatArray(value, head, tail, pathStack)
+		}
 	case yaml.AliasNode:
 		log.Debug("its an alias!")
 		DebugNode(value.Alias)
@@ -95,7 +100,7 @@ func (n *navigator) recurse(value *yaml.Node, head string, tail []string, pathSt
 	}
 }

-func (n *navigator) recurseMap(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
+func (n *navigator) recurseMap(value *yaml.Node, head string, tail []interface{}, pathStack []interface{}) error {
 	traversedEntry := false
 	errorVisiting := n.visitMatchingEntries(value, head, tail, pathStack, func(contents []*yaml.Node, indexInMap int) error {
 		log.Debug("recurseMap: visitMatchingEntries for %v", contents[indexInMap].Value)
@@ -125,18 +130,27 @@ func (n *navigator) recurseMap(value *yaml.Node, head string, tail []string, pat
 		return nil
 	}

+	_, errorParsingInt := strconv.ParseInt(head, 10, 64)
+
 	mapEntryKey := yaml.Node{Value: head, Kind: yaml.ScalarNode}
+
+	if errorParsingInt == nil {
+		// fixes a json encoding problem where keys that look like numbers
+		// get treated as numbers and cannot be used in a json map
+		mapEntryKey.Style = yaml.LiteralStyle
+	}
+
 	value.Content = append(value.Content, &mapEntryKey)
 	mapEntryValue := yaml.Node{Kind: guessKind(head, tail, 0)}
 	value.Content = append(value.Content, &mapEntryValue)
-	log.Debug("adding new node %v", head)
+	log.Debug("adding a new node %v - def a string", head)
 	return n.doTraverse(&mapEntryValue, head, tail, append(pathStack, head))
 }

 // need to pass the node in, as it may be aliased
 type mapVisitorFn func(contents []*yaml.Node, index int) error

-func (n *navigator) visitDirectMatchingEntries(node *yaml.Node, head string, tail []string, pathStack []interface{}, visit mapVisitorFn) error {
+func (n *navigator) visitDirectMatchingEntries(node *yaml.Node, head string, tail []interface{}, pathStack []interface{}, visit mapVisitorFn) error {
 	var contents = node.Content
 	for index := 0; index < len(contents); index = index + 2 {
 		content := contents[index]
@@ -151,7 +165,7 @@ func (n *navigator) visitDirectMatchingEntries(node *yaml.Node, head string, tai
 	return nil
 }

-func (n *navigator) visitMatchingEntries(node *yaml.Node, head string, tail []string, pathStack []interface{}, visit mapVisitorFn) error {
+func (n *navigator) visitMatchingEntries(node *yaml.Node, head string, tail []interface{}, pathStack []interface{}, visit mapVisitorFn) error {
 	var contents = node.Content
 	log.Debug("visitMatchingEntries %v", head)
 	DebugNode(node)
@@ -167,7 +181,7 @@ func (n *navigator) visitMatchingEntries(node *yaml.Node, head string, tail []st
 	return n.visitAliases(contents, head, tail, pathStack, visit)
 }

-func (n *navigator) visitAliases(contents []*yaml.Node, head string, tail []string, pathStack []interface{}, visit mapVisitorFn) error {
+func (n *navigator) visitAliases(contents []*yaml.Node, head string, tail []interface{}, pathStack []interface{}, visit mapVisitorFn) error {
 	// merge aliases are defined first, but we only want to traverse them
 	// if we don't find a match on this node first.
 	// traverse them backwards so that the last alias overrides the preceding.
@@ -198,7 +212,7 @@ func (n *navigator) visitAliases(contents []*yaml.Node, head string, tail []stri
 	return nil
 }

-func (n *navigator) visitAliasSequence(possibleAliasArray []*yaml.Node, head string, tail []string, pathStack []interface{}, visit mapVisitorFn) error {
+func (n *navigator) visitAliasSequence(possibleAliasArray []*yaml.Node, head string, tail []interface{}, pathStack []interface{}, visit mapVisitorFn) error {
 	// need to search this backwards too, so that aliases defined last override the preceding.
 	for aliasIndex := len(possibleAliasArray) - 1; aliasIndex >= 0; aliasIndex = aliasIndex - 1 {
 		child := possibleAliasArray[aliasIndex]
@@ -214,7 +228,7 @@ func (n *navigator) visitAliasSequence(possibleAliasArray []*yaml.Node, head str
 	return nil
 }

-func (n *navigator) splatArray(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
+func (n *navigator) splatArray(value *yaml.Node, head interface{}, tail []interface{}, pathStack []interface{}) error {
 	for index, childValue := range value.Content {
 		log.Debug("processing")
 		DebugNode(childValue)
@@ -234,14 +248,14 @@ func (n *navigator) splatArray(value *yaml.Node, head string, tail []string, pat
 	return nil
 }

-func (n *navigator) appendArray(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
+func (n *navigator) appendArray(value *yaml.Node, head interface{}, tail []interface{}, pathStack []interface{}) error {
 	var newNode = yaml.Node{Kind: guessKind(head, tail, 0)}
 	value.Content = append(value.Content, &newNode)
 	log.Debug("appending a new node, %v", value.Content)
 	return n.doTraverse(&newNode, head, tail, append(pathStack, len(value.Content)-1))
 }

-func (n *navigator) recurseArray(value *yaml.Node, index int64, head string, tail []string, pathStack []interface{}) error {
+func (n *navigator) recurseArray(value *yaml.Node, index int64, head interface{}, tail []interface{}, pathStack []interface{}) error {
 	for int64(len(value.Content)) <= index {
 		value.Content = append(value.Content, &yaml.Node{Kind: guessKind(head, tail, 0)})
 	}
--- a/pkg/yqlib/delete_navigation_strategy.go
+++ b/pkg/yqlib/delete_navigation_strategy.go
@@ -1,12 +1,10 @@
 package yqlib

 import (
-	"strconv"
-
 	yaml "gopkg.in/yaml.v3"
 )

-func DeleteNavigationStrategy(pathElementToDelete string) NavigationStrategy {
+func DeleteNavigationStrategy(pathElementToDelete interface{}) NavigationStrategy {
 	parser := NewPathParser()
 	return &NavigationStrategyImpl{
 		visitedNodes: []*NodeContext{},
@@ -34,12 +32,12 @@ func DeleteNavigationStrategy(pathElementToDelete string) NavigationStrategy {
 		},
 	}
 }
-func deleteFromMap(pathParser PathParser, contents []*yaml.Node, pathStack []interface{}, pathElementToDelete string) []*yaml.Node {
+func deleteFromMap(pathParser PathParser, contents []*yaml.Node, pathStack []interface{}, pathElementToDelete interface{}) []*yaml.Node {
 	newContents := make([]*yaml.Node, 0)
 	for index := 0; index < len(contents); index = index + 2 {
 		keyNode := contents[index]
 		valueNode := contents[index+1]
-		if !pathParser.MatchesNextPathElement(NewNodeContext(keyNode, pathElementToDelete, []string{}, pathStack), keyNode.Value) {
+		if !pathParser.MatchesNextPathElement(NewNodeContext(keyNode, pathElementToDelete, make([]interface{}, 0), pathStack), keyNode.Value) {
 			log.Debug("adding node %v", keyNode.Value)
 			newContents = append(newContents, keyNode, valueNode)
 		} else {
@@ -49,21 +47,23 @@ func deleteFromMap(pathParser PathParser, contents []*yaml.Node, pathStack []int
 	return newContents
 }

-func deleteFromArray(pathParser PathParser, content []*yaml.Node, pathStack []interface{}, pathElementToDelete string) []*yaml.Node {
+func deleteFromArray(pathParser PathParser, content []*yaml.Node, pathStack []interface{}, pathElementToDelete interface{}) []*yaml.Node {

-	var indexToDelete, err = strconv.ParseInt(pathElementToDelete, 10, 64) // nolint
-	if err == nil {
-		return deleteIndexInArray(content, indexToDelete)
-	}
-	log.Debug("%v is not a numeric index, finding matching patterns", pathElementToDelete)
-	var newArray = make([]*yaml.Node, 0)
+	switch pathElementToDelete := pathElementToDelete.(type) {
+	case int64:
+		return deleteIndexInArray(content, pathElementToDelete)
+	default:
+		log.Debug("%v is not a numeric index, finding matching patterns", pathElementToDelete)
+		var newArray = make([]*yaml.Node, 0)

-	for _, childValue := range content {
-		if !pathParser.MatchesNextPathElement(NewNodeContext(childValue, pathElementToDelete, []string{}, pathStack), childValue.Value) {
-			newArray = append(newArray, childValue)
+		for _, childValue := range content {
+			if !pathParser.MatchesNextPathElement(NewNodeContext(childValue, pathElementToDelete, make([]interface{}, 0), pathStack), childValue.Value) {
+				newArray = append(newArray, childValue)
+			}
 		}
+		return newArray
 	}
-	return newArray
+
 }

 func deleteIndexInArray(content []*yaml.Node, index int64) []*yaml.Node {
--- a/pkg/yqlib/lib.go
+++ b/pkg/yqlib/lib.go
@@ -43,7 +43,7 @@ func mergePathStackToString(pathStack []interface{}, appendArrays bool) string {
 	var sb strings.Builder
 	for index, path := range pathStack {
 		switch path.(type) {
-		case int:
+		case int, int64:
 			if appendArrays {
 				sb.WriteString("[+]")
 			} else {
@@ -52,13 +52,15 @@ func mergePathStackToString(pathStack []interface{}, appendArrays bool) string {

 		default:
 			s := fmt.Sprintf("%v", path)
+			var _, errParsingInt = strconv.ParseInt(s, 10, 64) // nolint
+
 			hasDot := strings.Contains(s, ".")
-			if hasDot {
-				sb.WriteString("[")
+			if hasDot || errParsingInt == nil {
+				sb.WriteString("\"")
 			}
 			sb.WriteString(s)
-			if hasDot {
-				sb.WriteString("]")
+			if hasDot || errParsingInt == nil {
+				sb.WriteString("\"")
 			}
 		}

@@ -66,34 +68,41 @@ func mergePathStackToString(pathStack []interface{}, appendArrays bool) string {
 			sb.WriteString(".")
 		}
 	}
-	return sb.String()
+	var pathString = sb.String()
+	log.Debug("got a path string: %v", pathString)
+	return pathString
 }

-func guessKind(head string, tail []string, guess yaml.Kind) yaml.Kind {
-	log.Debug("tail %v", tail)
+func guessKind(head interface{}, tail []interface{}, guess yaml.Kind) yaml.Kind {
+	log.Debug("guessKind: tail %v", tail)
 	if len(tail) == 0 && guess == 0 {
 		log.Debug("end of path, must be a scalar")
 		return yaml.ScalarNode
 	} else if len(tail) == 0 {
 		return guess
 	}
-
-	var _, errorParsingInt = strconv.ParseInt(tail[0], 10, 64)
-	if tail[0] == "+" || errorParsingInt == nil {
+	var next = tail[0]
+	switch next.(type) {
+	case int64:
 		return yaml.SequenceNode
+	default:
+		var nextString = fmt.Sprintf("%v", next)
+		if nextString == "+" {
+			return yaml.SequenceNode
+		}
+		pathParser := NewPathParser()
+		if (pathParser.IsPathExpression(nextString) || head == "**") && (guess == yaml.SequenceNode || guess == yaml.MappingNode) {
+			return guess
+		}
+		if guess == yaml.AliasNode {
+			log.Debug("guess was an alias, okey doke.")
+			return guess
+		}
+		log.Debug("forcing a mapping node")
+		log.Debug("yaml.SequenceNode %v", guess == yaml.SequenceNode)
+		log.Debug("yaml.ScalarNode %v", guess == yaml.ScalarNode)
+		return yaml.MappingNode
 	}
-	pathParser := NewPathParser()
-	if (pathParser.IsPathExpression(tail[0]) || head == "**") && (guess == yaml.SequenceNode || guess == yaml.MappingNode) {
-		return guess
-	}
-	if guess == yaml.AliasNode {
-		log.Debug("guess was an alias, okey doke.")
-		return guess
-	}
-	log.Debug("forcing a mapping node")
-	log.Debug("yaml.SequenceNode %v", guess == yaml.SequenceNode)
-	log.Debug("yaml.ScalarNode %v", guess == yaml.ScalarNode)
-	return yaml.MappingNode
 }

 type YqLib interface {
--- a/pkg/yqlib/navigation_strategy.go
+++ b/pkg/yqlib/navigation_strategy.go
@@ -8,13 +8,13 @@ import (

 type NodeContext struct {
 	Node      *yaml.Node
-	Head      string
-	Tail      []string
+	Head      interface{}
+	Tail      []interface{}
 	PathStack []interface{}
 }

-func NewNodeContext(node *yaml.Node, head string, tail []string, pathStack []interface{}) NodeContext {
-	newTail := make([]string, len(tail))
+func NewNodeContext(node *yaml.Node, head interface{}, tail []interface{}, pathStack []interface{}) NodeContext {
+	newTail := make([]interface{}, len(tail))
 	copy(newTail, tail)

 	newPathStack := make([]interface{}, len(pathStack))
--- a/pkg/yqlib/path_parser.go
+++ b/pkg/yqlib/path_parser.go
@@ -1,12 +1,13 @@
 package yqlib

 import (
+	"fmt"
 	"strconv"
 	"strings"
 )

 type PathParser interface {
-	ParsePath(path string) []string
+	ParsePath(path string) []interface{}
 	MatchesNextPathElement(nodeContext NodeContext, nodeKey string) bool
 	IsPathExpression(pathElement string) bool
 }
@@ -42,9 +43,11 @@ func (p *pathParser) MatchesNextPathElement(nodeContext NodeContext, nodeKey str
 	if head == "**" || head == "*" {
 		return true
 	}
-	if strings.Contains(head, "==") {
+	var headString = fmt.Sprintf("%v", head)
+
+	if strings.Contains(headString, "==") {
 		log.Debug("ooh deep recursion time")
-		result := strings.SplitN(head, "==", 2)
+		result := strings.SplitN(headString, "==", 2)
 		path := strings.TrimSpace(result[0])
 		value := strings.TrimSpace(result[1])
 		log.Debug("path %v", path)
@@ -70,17 +73,18 @@ func (p *pathParser) MatchesNextPathElement(nodeContext NodeContext, nodeKey str
 		}
 	}

-	return matchesString(head, nodeKey)
+	return matchesString(headString, nodeKey)
 }

-func (p *pathParser) ParsePath(path string) []string {
+func (p *pathParser) ParsePath(path string) []interface{} {
+	var paths = make([]interface{}, 0)
 	if path == "" {
-		return []string{}
+		return paths
 	}
-	return p.parsePathAccum([]string{}, path)
+	return p.parsePathAccum(paths, path)
 }

-func (p *pathParser) parsePathAccum(paths []string, remaining string) []string {
+func (p *pathParser) parsePathAccum(paths []interface{}, remaining string) []interface{} {
 	head, tail := p.nextYamlPath(remaining)
 	if tail == "" {
 		return append(paths, head)
@@ -88,11 +92,16 @@ func (p *pathParser) parsePathAccum(paths []string, remaining string) []string {
 	return p.parsePathAccum(append(paths, head), tail)
 }

-func (p *pathParser) nextYamlPath(path string) (pathElement string, remaining string) {
+func (p *pathParser) nextYamlPath(path string) (pathElement interface{}, remaining string) {
 	switch path[0] {
 	case '[':
 		// e.g [0].blah.cat -> we need to return "0" and "blah.cat"
-		return p.search(path[1:], []uint8{']'}, true)
+		var value, remainingBit = p.search(path[1:], []uint8{']'}, true)
+		var number, errParsingInt = strconv.ParseInt(value, 10, 64) // nolint
+		if errParsingInt == nil {
+			return number, remainingBit
+		}
+		return value, remainingBit
 	case '"':
 		// e.g "a.b".blah.cat -> we need to return "a.b" and "blah.cat"
 		return p.search(path[1:], []uint8{'"'}, true)
--- a/pkg/yqlib/path_parser_test.go
+++ b/pkg/yqlib/path_parser_test.go
@@ -10,20 +10,21 @@ var parser = NewPathParser()

 var parsePathsTests = []struct {
 	path          string
-	expectedPaths []string
+	expectedPaths []interface{}
 }{
-	{"a.b", []string{"a", "b"}},
-	{"a.b.**", []string{"a", "b", "**"}},
-	{"a.b.*", []string{"a", "b", "*"}},
-	{"a.b[0]", []string{"a", "b", "0"}},
-	{"a.b.d[+]", []string{"a", "b", "d", "+"}},
-	{"a", []string{"a"}},
-	{"a.b.c", []string{"a", "b", "c"}},
-	{"\"a.b\".c", []string{"a.b", "c"}},
-	{"a.\"b.c\".d", []string{"a", "b.c", "d"}},
-	{"[1].a.d", []string{"1", "a", "d"}},
-	{"a[0].c", []string{"a", "0", "c"}},
-	{"[0]", []string{"0"}},
+	{"a.b", append(make([]interface{}, 0), "a", "b")},
+	{"a.b.**", append(make([]interface{}, 0), "a", "b", "**")},
+	{"a.b.*", append(make([]interface{}, 0), "a", "b", "*")},
+	{"a.b[0]", append(make([]interface{}, 0), "a", "b", int64(0))},
+	{"a.b.0", append(make([]interface{}, 0), "a", "b", "0")},
+	{"a.b.d[+]", append(make([]interface{}, 0), "a", "b", "d", "+")},
+	{"a", append(make([]interface{}, 0), "a")},
+	{"a.b.c", append(make([]interface{}, 0), "a", "b", "c")},
+	{"\"a.b\".c", append(make([]interface{}, 0), "a.b", "c")},
+	{"a.\"b.c\".d", append(make([]interface{}, 0), "a", "b.c", "d")},
+	{"[1].a.d", append(make([]interface{}, 0), int64(1), "a", "d")},
+	{"a[0].c", append(make([]interface{}, 0), "a", int64(0), "c")},
+	{"[0]", append(make([]interface{}, 0), int64(0))},
 }

 func TestPathParserParsePath(t *testing.T) {