Added support for integer suffixes, e.g. 1k, 8mb

conf/lex.go

@@ -17,6 +17,7 @@ package conf
 
 import (
 	"fmt"
+	"unicode"
 	"unicode/utf8"
 )
 
@@ -182,7 +183,7 @@ func (lx *lexer) errorf(format string, values ...interface{}) stateFn {
 // lexTop consumes elements at the top level of data structure.
 func lexTop(lx *lexer) stateFn {
 	r := lx.next()
-	if isWhitespace(r) || isNL(r) {
+	if unicode.IsSpace(r) {
 		return lexSkip(lx, lexTop)
 	}
 
@@ -248,7 +249,7 @@ func lexKeyStart(lx *lexer) stateFn {
 	switch {
 	case isKeySeparator(r):
 		return lx.errorf("Unexpected key separator '%v'", r)
-	case isWhitespace(r) || isNL(r):
+	case unicode.IsSpace(r):
 		lx.next()
 		return lexSkip(lx, lexKeyStart)
 	case r == dqStringStart:
@@ -291,7 +292,7 @@ func lexQuotedKey(lx *lexer) stateFn {
 // is not whitespace) has already been consumed.
 func lexKey(lx *lexer) stateFn {
 	r := lx.peek()
-	if isWhitespace(r) || isNL(r) || isKeySeparator(r) || r == eof {
+	if unicode.IsSpace(r) || isKeySeparator(r) || r == eof {
 		lx.emit(itemKey)
 		return lexKeyEnd
 	}
@@ -305,7 +306,7 @@ func lexKey(lx *lexer) stateFn {
 func lexKeyEnd(lx *lexer) stateFn {
 	r := lx.next()
 	switch {
-	case isWhitespace(r) || isNL(r):
+	case unicode.IsSpace(r):
 		return lexSkip(lx, lexKeyEnd)
 	case isKeySeparator(r):
 		return lexSkip(lx, lexValue)
@@ -345,11 +346,11 @@ func lexValue(lx *lexer) stateFn {
 		lx.ignore() // ignore the " or '
 		return lexDubQuotedString
 	case r == '-':
-		return lexNumberStart
+		return lexNegNumberStart
 	case r == blockStart:
 		lx.ignore()
 		return lexBlock
-	case isDigit(r):
+	case unicode.IsDigit(r):
 		lx.backup() // avoid an extra state and use the same as above
 		return lexNumberOrDateOrIPStart
 	case r == '.': // special error case, be kind to users
@@ -366,7 +367,7 @@ func lexValue(lx *lexer) stateFn {
 func lexArrayValue(lx *lexer) stateFn {
 	r := lx.next()
 	switch {
-	case isWhitespace(r) || isNL(r):
+	case unicode.IsSpace(r):
 		return lexSkip(lx, lexArrayValue)
 	case r == commentHashStart:
 		lx.push(lexArrayValue)
@@ -433,7 +434,7 @@ func lexMapKeyStart(lx *lexer) stateFn {
 	switch {
 	case isKeySeparator(r):
 		return lx.errorf("Unexpected key separator '%v'.", r)
-	case isWhitespace(r) || isNL(r):
+	case unicode.IsSpace(r):
 		lx.next()
 		return lexSkip(lx, lexMapKeyStart)
 	case r == mapEnd:
@@ -491,7 +492,7 @@ func lexMapDubQuotedKey(lx *lexer) stateFn {
 // is not whitespace) has already been consumed.
 func lexMapKey(lx *lexer) stateFn {
 	r := lx.peek()
-	if isWhitespace(r) || isNL(r) || isKeySeparator(r) {
+	if unicode.IsSpace(r) || isKeySeparator(r) {
 		lx.emit(itemKey)
 		return lexMapKeyEnd
 	}
@@ -505,7 +506,7 @@ func lexMapKey(lx *lexer) stateFn {
 func lexMapKeyEnd(lx *lexer) stateFn {
 	r := lx.next()
 	switch {
-	case isWhitespace(r) || isNL(r):
+	case unicode.IsSpace(r):
 		return lexSkip(lx, lexMapKeyEnd)
 	case isKeySeparator(r):
 		return lexSkip(lx, lexMapValue)
@@ -521,7 +522,7 @@ func lexMapKeyEnd(lx *lexer) stateFn {
 func lexMapValue(lx *lexer) stateFn {
 	r := lx.next()
 	switch {
-	case isWhitespace(r) || isNL(r):
+	case unicode.IsSpace(r):
 		return lexSkip(lx, lexMapValue)
 	case r == mapValTerm:
 		return lx.errorf("Unexpected map value terminator %q.", mapValTerm)
@@ -722,7 +723,7 @@ func lexStringBinary(lx *lexer) stateFn {
 // It assumes that NO negative sign has been consumed, that is triggered above.
 func lexNumberOrDateOrIPStart(lx *lexer) stateFn {
 	r := lx.next()
-	if !isDigit(r) {
+	if !unicode.IsDigit(r) {
 		if r == '.' {
 			return lx.errorf("Floats must start with a digit, not '.'.")
 		}
@@ -740,10 +741,12 @@ func lexNumberOrDateOrIP(lx *lexer) stateFn {
 			return lx.errorf("All ISO8601 dates must be in full Zulu form.")
 		}
 		return lexDateAfterYear
-	case isDigit(r):
+	case unicode.IsDigit(r):
 		return lexNumberOrDateOrIP
 	case r == '.':
-		return lexFloatStart
+		return lexFloatStart // Assume float at first, but could be IP
+	case isNumberSuffix(r):
+		return lexConvenientNumber
 	}
 
 	lx.backup()
@@ -751,6 +754,18 @@ func lexNumberOrDateOrIP(lx *lexer) stateFn {
 	return lx.pop()
 }
 
+// lexConvenientNumber is when we have a suffix, e.g. 1k or 1Mb
+func lexConvenientNumber(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case r == 'b' || r == 'B':
+		return lexConvenientNumber
+	}
+	lx.backup()
+	lx.emit(itemInteger)
+	return lx.pop()
+}
+
 // lexDateAfterYear consumes a full Zulu Datetime in ISO8601 format.
 // It assumes that "YYYY-" has already been consumed.
 func lexDateAfterYear(lx *lexer) stateFn {
@@ -765,7 +780,7 @@ func lexDateAfterYear(lx *lexer) stateFn {
 	for _, f := range formats {
 		r := lx.next()
 		if f == '0' {
-			if !isDigit(r) {
+			if !unicode.IsDigit(r) {
 				return lx.errorf("Expected digit in ISO8601 datetime, "+
 					"but found '%v' instead.", r)
 			}
@@ -778,29 +793,31 @@ func lexDateAfterYear(lx *lexer) stateFn {
 	return lx.pop()
 }
 
-// lexNumberStart consumes either an integer or a float. It assumes that a
+// lexNegNumberStart consumes either an integer or a float. It assumes that a
 // negative sign has already been read, but that *no* digits have been consumed.
-// lexNumberStart will move to the appropriate integer or float states.
+// lexNegNumberStart will move to the appropriate integer or float states.
-func lexNumberStart(lx *lexer) stateFn {
+func lexNegNumberStart(lx *lexer) stateFn {
 	// we MUST see a digit. Even floats have to start with a digit.
 	r := lx.next()
-	if !isDigit(r) {
+	if !unicode.IsDigit(r) {
 		if r == '.' {
 			return lx.errorf("Floats must start with a digit, not '.'.")
 		}
 		return lx.errorf("Expected a digit but got '%v'.", r)
 	}
-	return lexNumber
+	return lexNegNumber
 }
 
-// lexNumber consumes an integer or a float after seeing the first digit.
+// lexNumber consumes a negative integer or a float after seeing the first digit.
-func lexNumber(lx *lexer) stateFn {
+func lexNegNumber(lx *lexer) stateFn {
 	r := lx.next()
 	switch {
-	case isDigit(r):
+	case unicode.IsDigit(r):
-		return lexNumber
+		return lexNegNumber
 	case r == '.':
 		return lexFloatStart
+	case isNumberSuffix(r):
+		return lexConvenientNumber
 	}
 	lx.backup()
 	lx.emit(itemInteger)
@@ -811,7 +828,7 @@ func lexNumber(lx *lexer) stateFn {
 // Namely, at least one digit is required.
 func lexFloatStart(lx *lexer) stateFn {
 	r := lx.next()
-	if !isDigit(r) {
+	if !unicode.IsDigit(r) {
 		return lx.errorf("Floats must have a digit after the '.', but got "+
 			"'%v' instead.", r)
 	}
@@ -822,7 +839,7 @@ func lexFloatStart(lx *lexer) stateFn {
 // Assumes that one digit has been consumed after a '.' already.
 func lexFloat(lx *lexer) stateFn {
 	r := lx.next()
-	if isDigit(r) {
+	if unicode.IsDigit(r) {
 		return lexFloat
 	}
 
@@ -839,7 +856,7 @@ func lexFloat(lx *lexer) stateFn {
 // lexIPAddr consumes IP addrs, like 127.0.0.1:4222
 func lexIPAddr(lx *lexer) stateFn {
 	r := lx.next()
-	if isDigit(r) || r == '.' || r == ':' {
+	if unicode.IsDigit(r) || r == '.' || r == ':' {
 		return lexIPAddr
 	}
 	lx.backup()
@@ -876,6 +893,11 @@ func lexSkip(lx *lexer, nextState stateFn) stateFn {
 	}
 }
 
+// Tests to see if we have a number suffix
+func isNumberSuffix(r rune) bool {
+	return r == 'k' || r == 'K' || r == 'm' || r == 'M' || r == 'g' || r == 'G'
+}
+
 // Tests for both key separators
 func isKeySeparator(r rune) bool {
 	return r == keySepEqual || r == keySepColon
@@ -891,10 +913,6 @@ func isNL(r rune) bool {
 	return r == '\n' || r == '\r'
 }
 
-func isDigit(r rune) bool {
-	return r >= '0' && r <= '9'
-}
-
 func isHexadecimal(r rune) bool {
 	return (r >= '0' && r <= '9') ||
 		(r >= 'a' && r <= 'f') ||

conf/lex_test.go

@@ -100,6 +100,89 @@ func TestSimpleKeyNegativeIntegerValues(t *testing.T) {
 	expect(t, lx, expectedItems)
 }
 
+func TestConvenientIntegerValues(t *testing.T) {
+	expectedItems := []item{
+		{itemKey, "foo", 1},
+		{itemInteger, "1k", 1},
+		{itemEOF, "", 1},
+	}
+	lx := lex("foo = 1k")
+	expect(t, lx, expectedItems)
+
+	expectedItems = []item{
+		{itemKey, "foo", 1},
+		{itemInteger, "1K", 1},
+		{itemEOF, "", 1},
+	}
+	lx = lex("foo = 1K")
+	expect(t, lx, expectedItems)
+
+	expectedItems = []item{
+		{itemKey, "foo", 1},
+		{itemInteger, "1m", 1},
+		{itemEOF, "", 1},
+	}
+	lx = lex("foo = 1m")
+	expect(t, lx, expectedItems)
+
+	expectedItems = []item{
+		{itemKey, "foo", 1},
+		{itemInteger, "1M", 1},
+		{itemEOF, "", 1},
+	}
+	lx = lex("foo = 1M")
+	expect(t, lx, expectedItems)
+
+	expectedItems = []item{
+		{itemKey, "foo", 1},
+		{itemInteger, "1g", 1},
+		{itemEOF, "", 1},
+	}
+	lx = lex("foo = 1g")
+	expect(t, lx, expectedItems)
+
+	expectedItems = []item{
+		{itemKey, "foo", 1},
+		{itemInteger, "1G", 1},
+		{itemEOF, "", 1},
+	}
+	lx = lex("foo = 1G")
+	expect(t, lx, expectedItems)
+
+	expectedItems = []item{
+		{itemKey, "foo", 1},
+		{itemInteger, "1MB", 1},
+		{itemEOF, "", 1},
+	}
+	lx = lex("foo = 1MB")
+	expect(t, lx, expectedItems)
+
+	expectedItems = []item{
+		{itemKey, "foo", 1},
+		{itemInteger, "1Gb", 1},
+		{itemEOF, "", 1},
+	}
+	lx = lex("foo = 1Gb")
+	expect(t, lx, expectedItems)
+
+	// Negative versions
+	expectedItems = []item{
+		{itemKey, "foo", 1},
+		{itemInteger, "-1m", 1},
+		{itemEOF, "", 1},
+	}
+	lx = lex("foo = -1m")
+	expect(t, lx, expectedItems)
+
+	expectedItems = []item{
+		{itemKey, "foo", 1},
+		{itemInteger, "-1GB", 1},
+		{itemEOF, "", 1},
+	}
+	lx = lex("foo = -1GB ")
+	expect(t, lx, expectedItems)
+}
+
 func TestSimpleKeyFloatValues(t *testing.T) {
 	expectedItems := []item{
 		{itemKey, "foo", 1},

conf/parse.go

@@ -20,6 +20,7 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"unicode"
 )
 
 type parser struct {
@@ -117,7 +118,15 @@ func (p *parser) processItem(it item) error {
 	case itemString:
 		p.setValue(it.val) // FIXME(dlc) sanitize string?
 	case itemInteger:
-		num, err := strconv.ParseInt(it.val, 10, 64)
+		lastDigit := 0
+		for _, r := range it.val {
+			if !unicode.IsDigit(r) {
+				break
+			}
+			lastDigit++
+		}
+		numStr := it.val[:lastDigit]
+		num, err := strconv.ParseInt(numStr, 10, 64)
 		if err != nil {
 			if e, ok := err.(*strconv.NumError); ok &&
 				e.Err == strconv.ErrRange {
@@ -125,7 +134,25 @@ func (p *parser) processItem(it item) error {
 			}
 			return fmt.Errorf("Expected integer, but got '%s'.", it.val)
 		}
-		p.setValue(num)
+		// Process a suffix
+		suffix := strings.ToLower(strings.TrimSpace(it.val[lastDigit:]))
+		switch suffix {
+		case "":
+			p.setValue(num)
+		case "k":
+			p.setValue(num * 1000)
+		case "kb":
+			p.setValue(num * 1024)
+		case "m":
+			p.setValue(num * 1000 * 1000)
+		case "mb":
+			p.setValue(num * 1024 * 1024)
+		case "g":
+			p.setValue(num * 1000 * 1000 * 1000)
+		case "gb":
+			p.setValue(num * 1024 * 1024 * 1024)
+		}
+
 	case itemFloat:
 		num, err := strconv.ParseFloat(it.val, 64)
 		if err != nil {

conf/parse_test.go

@@ -96,6 +96,27 @@ func TestBcryptVariable(t *testing.T) {
 	test(t, "password: $2a$11$ooo", ex)
 }
 
+var easynum = `
+k = 8k
+kb = 4kb
+m = 1m
+mb = 2MB
+g = 2g
+gb = 22GB
+`
+
+func TestConvenientNumbers(t *testing.T) {
+	ex := map[string]interface{}{
+		"k":  int64(8 * 1000),
+		"kb": int64(4 * 1024),
+		"m":  int64(1000 * 1000),
+		"mb": int64(2 * 1024 * 1024),
+		"g":  int64(2 * 1000 * 1000 * 1000),
+		"gb": int64(22 * 1024 * 1024 * 1024),
+	}
+	test(t, easynum, ex)
+}
+
 var sample1 = `
 foo  {
   host {