diff --git a/conf/lex.go b/conf/lex.go index 290ac13c..cec332b7 100644 --- a/conf/lex.go +++ b/conf/lex.go @@ -1,4 +1,4 @@ -// Copyright 2013 Apcera Inc. All rights reserved. +// Copyright 2013-2015 Apcera Inc. All rights reserved. // Customized heavily from // https://github.com/BurntSushi/toml/blob/master/lex.go, which is based on @@ -154,15 +154,6 @@ func (lx *lexer) backup() { } } -// accept consumes the next rune if it's equal to `valid`. -func (lx *lexer) accept(valid rune) bool { - if lx.next() == valid { - return true - } - lx.backup() - return false -} - // peek returns but does not consume the next rune in the input. func (lx *lexer) peek() rune { r := lx.next() @@ -362,8 +353,8 @@ func lexValue(lx *lexer) stateFn { case isNL(r): return lx.errorf("Expected value but found new line") } + lx.backup() return lexString - //return lx.errorf("Expected value but found '%s' instead.", r) } // lexArrayValue consumes one value in an array. It assumes that '[' or ',' @@ -529,17 +520,6 @@ func lexMapValue(lx *lexer) stateFn { switch { case isWhitespace(r) || isNL(r): return lexSkip(lx, lexMapValue) - case r == commentHashStart: - lx.push(lexMapValue) - return lexCommentStart - case r == commentSlashStart: - rn := lx.next() - if rn == commentSlashStart { - lx.push(lexMapValue) - return lexCommentStart - } - lx.backup() - fallthrough case r == mapValTerm: return lx.errorf("Unexpected map value terminator %q.", mapValTerm) case r == mapEnd: @@ -649,32 +629,6 @@ func lexString(lx *lexer) stateFn { return lexString } -// lexDubString consumes the inner contents of a string. It assumes that the -// beginning '"' has already been consumed and ignored. -func lexDubString(lx *lexer) stateFn { - r := lx.next() - switch { - case r == '\\': - return lexStringEscape - // Termination of non-quoted strings - case isNL(r) || r == eof || r == optValTerm || isWhitespace(r): - lx.backup() - if lx.isBool() { - lx.emit(itemBool) - } else { - lx.emit(itemString) - } - return lx.pop() - case r == dqStringEnd: - lx.backup() - lx.emit(itemString) - lx.next() - lx.ignore() - return lx.pop() - } - return lexDubString -} - // lexBlock consumes the inner contents as a string. It assumes that the // beginning '(' has already been consumed and ignored. It will continue // processing until it finds a ')' on a new line by itself. diff --git a/conf/lex_test.go b/conf/lex_test.go index 71e427ed..bd917613 100644 --- a/conf/lex_test.go +++ b/conf/lex_test.go @@ -38,6 +38,29 @@ func TestSimpleKeyStringValues(t *testing.T) { // NL lx = lex("foo='bar'\r\n") expect(t, lx, expectedItems) + lx = lex("foo=\t'bar'\t") + expect(t, lx, expectedItems) +} + +func TestComplexStringValues(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1}, + {itemString, "bar\\r\\n \\t", 1}, + {itemEOF, "", 2}, + } + + lx := lex("foo = 'bar\\r\\n \\t'") + expect(t, lx, expectedItems) +} + +func TestBinaryString(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1}, + {itemString, "\\x22", 1}, + {itemEOF, "", 1}, + } + lx := lex("foo = \\x22") + expect(t, lx, expectedItems) } func TestSimpleKeyIntegerValues(t *testing.T) { @@ -96,6 +119,21 @@ func TestComments(t *testing.T) { expect(t, lx, expectedItems) } +func TestTopValuesWithComments(t *testing.T) { + expectedItems := []item{ + {itemKey, "foo", 1}, + {itemInteger, "123", 1}, + {itemCommentStart, "", 1}, + {itemText, " This is a comment", 1}, + {itemEOF, "", 1}, + } + + lx := lex("foo = 123 // This is a comment") + expect(t, lx, expectedItems) + lx = lex("foo=123 # This is a comment") + expect(t, lx, expectedItems) +} + func TestArrays(t *testing.T) { expectedItems := []item{ {itemKey, "foo", 1}, @@ -153,7 +191,7 @@ func TestMultilineArrays(t *testing.T) { var mlArrayNoSep = ` # top level comment foo = [ - 1 + 1 // foo 2 3 'bar' @@ -168,6 +206,8 @@ func TestMultilineArraysNoSep(t *testing.T) { {itemKey, "foo", 3}, {itemArrayStart, "", 3}, {itemInteger, "1", 4}, + {itemCommentStart, "", 4}, + {itemText, " foo", 4}, {itemInteger, "2", 5}, {itemInteger, "3", 6}, {itemString, "bar", 7}, @@ -197,8 +237,8 @@ func TestSimpleMap(t *testing.T) { var mlMap = ` foo = { - ip = '127.0.0.1' - port= 4242 + ip = '127.0.0.1' # the IP + port= 4242 // the port } ` @@ -208,8 +248,12 @@ func TestMultilineMap(t *testing.T) { {itemMapStart, "", 2}, {itemKey, "ip", 3}, {itemString, "127.0.0.1", 3}, + {itemCommentStart, "", 3}, + {itemText, " the IP", 3}, {itemKey, "port", 4}, {itemInteger, "4242", 4}, + {itemCommentStart, "", 4}, + {itemText, " the port", 4}, {itemMapEnd, "", 5}, {itemEOF, "", 5}, }