diff --git a/conf/lex.go b/conf/lex.go index 87fe466b..b50c0ffe 100644 --- a/conf/lex.go +++ b/conf/lex.go @@ -57,6 +57,8 @@ const ( sqStringStart = '\'' sqStringEnd = '\'' optValTerm = ';' + blockStart = '(' + blockEnd = ')' ) type stateFn func(lx *lexer) stateFn @@ -257,7 +259,10 @@ func lexKeyStart(lx *lexer) stateFn { case isWhitespace(r) || isNL(r): lx.next() return lexSkip(lx, lexKeyStart) - case r == sqStringStart || r == dqStringStart: + case r == dqStringStart: + lx.next() + return lexSkip(lx, lexDubQuotedKey) + case r == sqStringStart: lx.next() return lexSkip(lx, lexQuotedKey) } @@ -266,10 +271,22 @@ func lexKeyStart(lx *lexer) stateFn { return lexKey } +// lexDubQuotedKey consumes the text of a key between quotes. +func lexDubQuotedKey(lx *lexer) stateFn { + r := lx.peek() + if r == dqStringEnd { + lx.emit(itemKey) + lx.next() + return lexSkip(lx, lexKeyEnd) + } + lx.next() + return lexDubQuotedKey +} + // lexQuotedKey consumes the text of a key between quotes. func lexQuotedKey(lx *lexer) stateFn { r := lx.peek() - if r == sqStringEnd || r == dqStringEnd { + if r == sqStringEnd { lx.emit(itemKey) lx.next() return lexSkip(lx, lexKeyEnd) @@ -326,11 +343,17 @@ func lexValue(lx *lexer) stateFn { lx.ignore() lx.emit(itemMapStart) return lexMapKeyStart - case r == dqStringStart || r == sqStringStart: + case r == sqStringStart: lx.ignore() // ignore the " or ' return lexQuotedString + case r == dqStringStart: + lx.ignore() // ignore the " or ' + return lexDubQuotedString case r == '-': return lexNumberStart + case r == blockStart: + lx.ignore() + return lexBlock case isDigit(r): lx.backup() // avoid an extra state and use the same as above return lexNumberOrDateStart @@ -434,9 +457,12 @@ func lexMapKeyStart(lx *lexer) stateFn { return lexCommentStart } lx.backup() - case r == sqStringStart || r == dqStringStart: + case r == sqStringStart: lx.next() return lexSkip(lx, lexMapQuotedKey) + case r == dqStringStart: + lx.next() + return lexSkip(lx, lexMapDubQuotedKey) } lx.ignore() lx.next() @@ -446,7 +472,7 @@ func lexMapKeyStart(lx *lexer) stateFn { // lexMapQuotedKey consumes the text of a key between quotes. func lexMapQuotedKey(lx *lexer) stateFn { r := lx.peek() - if r == sqStringEnd || r == dqStringEnd { + if r == sqStringEnd { lx.emit(itemKey) lx.next() return lexSkip(lx, lexMapKeyEnd) @@ -455,6 +481,18 @@ func lexMapQuotedKey(lx *lexer) stateFn { return lexMapQuotedKey } +// lexMapQuotedKey consumes the text of a key between quotes. +func lexMapDubQuotedKey(lx *lexer) stateFn { + r := lx.peek() + if r == dqStringEnd { + lx.emit(itemKey) + lx.next() + return lexSkip(lx, lexMapKeyEnd) + } + lx.next() + return lexMapDubQuotedKey +} + // lexMapKey consumes the text of a key. Assumes that the first character (which // is not whitespace) has already been consumed. func lexMapKey(lx *lexer) stateFn { @@ -559,7 +597,7 @@ func (lx *lexer) isBool() bool { func lexQuotedString(lx *lexer) stateFn { r := lx.next() switch { - case r == dqStringEnd || r == sqStringEnd: + case r == sqStringEnd: lx.backup() lx.emit(itemString) lx.next() @@ -569,6 +607,22 @@ func lexQuotedString(lx *lexer) stateFn { return lexQuotedString } +// lexDubQuotedString consumes the inner contents of a string. It assumes that the +// beginning '"' has already been consumed and ignored. It will not interpret any +// internal contents. +func lexDubQuotedString(lx *lexer) stateFn { + r := lx.next() + switch { + case r == dqStringEnd: + lx.backup() + lx.emit(itemString) + lx.next() + lx.ignore() + return lx.pop() + } + return lexDubQuotedString +} + // lexString consumes the inner contents of a string. It assumes that the // beginning '"' has already been consumed and ignored. func lexString(lx *lexer) stateFn { @@ -585,7 +639,7 @@ func lexString(lx *lexer) stateFn { lx.emit(itemString) } return lx.pop() - case r == dqStringEnd || r == sqStringEnd: + case r == sqStringEnd: lx.backup() lx.emit(itemString) lx.next() @@ -595,6 +649,66 @@ func lexString(lx *lexer) stateFn { return lexString } +// lexDubString consumes the inner contents of a string. It assumes that the +// beginning '"' has already been consumed and ignored. +func lexDubString(lx *lexer) stateFn { + r := lx.next() + switch { + case r == '\\': + return lexStringEscape + // Termination of non-quoted strings + case isNL(r) || r == eof || r == optValTerm || isWhitespace(r): + lx.backup() + if lx.isBool() { + lx.emit(itemBool) + } else { + lx.emit(itemString) + } + return lx.pop() + case r == dqStringEnd: + lx.backup() + lx.emit(itemString) + lx.next() + lx.ignore() + return lx.pop() + } + return lexDubString +} + +// lexBlock consumes the inner contents as a string. It assumes that the +// beginning '(' has already been consumed and ignored. It will continue +// processing until it finds a ')' on a new line by itself. +func lexBlock(lx *lexer) stateFn { + r := lx.next() + switch { + case r == blockEnd: + lx.backup() + lx.backup() + + // Looking for a ')' character on a line by itself, if the previous + // character isn't a new line, then break so we keep processing the block. + if lx.next() != '\n' { + lx.next() + break + } + lx.next() + + // Make sure the next character is a new line or an eof. We want a ')' on a + // bare line by itself. + switch lx.next() { + case '\n', eof: + lx.backup() + lx.backup() + lx.emit(itemString) + lx.next() + lx.ignore() + return lx.pop() + } + lx.backup() + } + return lexBlock +} + // lexStringEscape consumes an escaped character. It assumes that the preceding // '\\' has already been consumed. func lexStringEscape(lx *lexer) stateFn { diff --git a/conf/lex_test.go b/conf/lex_test.go index a2848337..71e427ed 100644 --- a/conf/lex_test.go +++ b/conf/lex_test.go @@ -14,7 +14,7 @@ func expect(t *testing.T, lx *lexer, items []item) { t.Fatal(item.val) } if item != items[i] { - t.Fatalf("Testing: '%s'\nExpected %+v, received %+v\n", + t.Fatalf("Testing: '%s'\nExpected %q, received %q\n", lx.input, items[i], item) } } @@ -482,4 +482,46 @@ func TestDoubleNestedMapsNewLines(t *testing.T) { expect(t, lx, expectedItems) } +var blockexample = ` +numbers ( +1234567890 +) +` +func TestBlockString(t *testing.T) { + expectedItems := []item{ + {itemKey, "numbers", 2}, + {itemString, "\n1234567890\n", 4}, + } + lx := lex(blockexample) + expect(t, lx, expectedItems) +} + +func TestBlockStringEOF(t *testing.T) { + expectedItems := []item{ + {itemKey, "numbers", 2}, + {itemString, "\n1234567890\n", 4}, + } + blockbytes := []byte(blockexample[0 : len(blockexample)-1]) + blockbytes = append(blockbytes, 0) + lx := lex(string(blockbytes)) + expect(t, lx, expectedItems) +} + +var mlblockexample = ` +numbers ( + 12(34)56 + ( + 7890 + ) +) +` + +func TestBlockStringMultiLine(t *testing.T) { + expectedItems := []item{ + {itemKey, "numbers", 2}, + {itemString, "\n 12(34)56\n (\n 7890\n )\n", 7}, + } + lx := lex(mlblockexample) + expect(t, lx, expectedItems) +} diff --git a/conf/parse_test.go b/conf/parse_test.go index 3bceb2a0..1e2626f8 100644 --- a/conf/parse_test.go +++ b/conf/parse_test.go @@ -95,3 +95,21 @@ func TestSample2(t *testing.T) { test(t, cluster, ex) } + +var sample3 = ` +foo { + expr = '(true == "false")' + text = 'This is a multi-line +text block.' +} +` + +func TestSample3(t *testing.T) { + ex := map[string]interface{}{ + "foo": map[string]interface{}{ + "expr": "(true == \"false\")", + "text": "This is a multi-line\ntext block.", + }, + } + test(t, sample3, ex) +}