Add support for 'include' to configuration files

conf/includes/passwords.conf

@@ -0,0 +1,3 @@
+# Just foo & bar for testing
+ALICE_PASS: $2a$10$UHR6GhotWhpLsKtVP0/i6.Nh9.fuY73cWjLoJjb2sKT8KISBcUW5q
+BOB_PASS: $2a$11$dZM98SpGeI7dCFFGSpt.JObQcix8YHml4TBUZoge9R1uxnMIln5ly

conf/includes/users.conf

@@ -0,0 +1,8 @@
+# Users configuration
+
+include ./passwords.conf;
+
+users = [
+  {user: alice, password: $ALICE_PASS}
+  {user: bob,   password: $BOB_PASS}
+]

conf/lex.go

@@ -17,6 +17,7 @@ package conf
 
 import (
 	"fmt"
+	"strings"
 	"unicode"
 	"unicode/utf8"
 )
@@ -40,6 +41,7 @@ const (
 	itemMapEnd
 	itemCommentStart
 	itemVariable
+	itemInclude
 )
 
 const (
@@ -288,11 +290,118 @@ func lexQuotedKey(lx *lexer) stateFn {
 	return lexQuotedKey
 }
 
+// keyCheckKeyword will check for reserved keywords as the key value when the key is
+// separated with a space.
+func (lx *lexer) keyCheckKeyword(fallThrough, push stateFn) stateFn {
+	key := strings.ToLower(lx.input[lx.start:lx.pos])
+	switch key {
+	case "include":
+		lx.ignore()
+		if push != nil {
+			lx.push(push)
+		}
+		return lexIncludeStart
+	}
+	lx.emit(itemKey)
+	return fallThrough
+}
+
+// lexIncludeStart will consume the whitespace til the start of the value.
+func lexIncludeStart(lx *lexer) stateFn {
+	r := lx.next()
+	if isWhitespace(r) {
+		return lexSkip(lx, lexIncludeStart)
+	}
+	lx.backup()
+	return lexInclude
+}
+
+// lexIncludeQuotedString consumes the inner contents of a string. It assumes that the
+// beginning '"' has already been consumed and ignored. It will not interpret any
+// internal contents.
+func lexIncludeQuotedString(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case r == sqStringEnd:
+		lx.backup()
+		lx.emit(itemInclude)
+		lx.next()
+		lx.ignore()
+		return lx.pop()
+	}
+	return lexIncludeQuotedString
+}
+
+// lexIncludeDubQuotedString consumes the inner contents of a string. It assumes that the
+// beginning '"' has already been consumed and ignored. It will not interpret any
+// internal contents.
+func lexIncludeDubQuotedString(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case r == dqStringEnd:
+		lx.backup()
+		lx.emit(itemInclude)
+		lx.next()
+		lx.ignore()
+		return lx.pop()
+	}
+	return lexIncludeDubQuotedString
+}
+
+// lexIncludeString consumes the inner contents of a raw string.
+func lexIncludeString(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case isNL(r) || r == eof || r == optValTerm || r == mapEnd || isWhitespace(r):
+		lx.backup()
+		lx.emit(itemInclude)
+		return lx.pop()
+	case r == sqStringEnd:
+		lx.backup()
+		lx.emit(itemInclude)
+		lx.next()
+		lx.ignore()
+		return lx.pop()
+	}
+	return lexIncludeString
+}
+
+// lexInclude will consume the include value.
+func lexInclude(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case r == sqStringStart:
+		lx.ignore() // ignore the " or '
+		return lexIncludeQuotedString
+	case r == dqStringStart:
+		lx.ignore() // ignore the " or '
+		return lexIncludeDubQuotedString
+	case r == arrayStart:
+		return lx.errorf("Expected include value but found start of an array")
+	case r == mapStart:
+		return lx.errorf("Expected include value but found start of a map")
+	case r == blockStart:
+		return lx.errorf("Expected include value but found start of a block")
+	case unicode.IsDigit(r), r == '-':
+		return lx.errorf("Expected include value but found start of a number")
+	case r == '\\':
+		return lx.errorf("Expected include value but found escape sequence")
+	case isNL(r):
+		return lx.errorf("Expected include value but found new line")
+	}
+	lx.backup()
+	return lexIncludeString
+}
+
 // lexKey consumes the text of a key. Assumes that the first character (which
 // is not whitespace) has already been consumed.
 func lexKey(lx *lexer) stateFn {
 	r := lx.peek()
-	if unicode.IsSpace(r) || isKeySeparator(r) || r == eof {
+	if unicode.IsSpace(r) {
+		// Spaces signal we could be looking at a keyword, e.g. include.
+		// Keywords will eat the keyword and set the appropriate return stateFn.
+		return lx.keyCheckKeyword(lexKeyEnd, nil)
+	} else if isKeySeparator(r) || r == eof {
 		lx.emit(itemKey)
 		return lexKeyEnd
 	}
@@ -492,7 +601,11 @@ func lexMapDubQuotedKey(lx *lexer) stateFn {
 // is not whitespace) has already been consumed.
 func lexMapKey(lx *lexer) stateFn {
 	r := lx.peek()
-	if unicode.IsSpace(r) || isKeySeparator(r) {
+	if unicode.IsSpace(r) {
+		// Spaces signal we could be looking at a keyword, e.g. include.
+		// Keywords will eat the keyword and set the appropriate return stateFn.
+		return lx.keyCheckKeyword(lexMapKeyEnd, lexMapValueEnd)
+	} else if isKeySeparator(r) {
 		lx.emit(itemKey)
 		return lexMapKeyEnd
 	}
@@ -953,6 +1066,8 @@ func (itype itemType) String() string {
 		return "CommentStart"
 	case itemVariable:
 		return "Variable"
+	case itemInclude:
+		return "Include"
 	}
 	panic(fmt.Sprintf("BUG: Unknown type '%s'.", itype.String()))
 }

conf/lex_test.go

@@ -842,3 +842,40 @@ func TestArrayOfMaps(t *testing.T) {
 	lx := lex(arrayOfMaps)
 	expect(t, lx, expectedItems)
 }
+
+func TestInclude(t *testing.T) {
+	expectedItems := []item{
+		{itemInclude, "users.conf", 1},
+		{itemEOF, "", 1},
+	}
+	lx := lex("include \"users.conf\"")
+	expect(t, lx, expectedItems)
+
+	lx = lex("include 'users.conf'")
+	expect(t, lx, expectedItems)
+
+	lx = lex("include users.conf")
+	expect(t, lx, expectedItems)
+}
+
+func TestMapInclude(t *testing.T) {
+	expectedItems := []item{
+		{itemKey, "foo", 1},
+		{itemMapStart, "", 1},
+		{itemInclude, "users.conf", 1},
+		{itemMapEnd, "", 1},
+		{itemEOF, "", 1},
+	}
+
+	lx := lex("foo { include users.conf }")
+	expect(t, lx, expectedItems)
+
+	lx = lex("foo {include users.conf}")
+	expect(t, lx, expectedItems)
+
+	lx = lex("foo { include 'users.conf' }")
+	expect(t, lx, expectedItems)
+
+	lx = lex("foo { include \"users.conf\"}")
+	expect(t, lx, expectedItems)
+}

conf/parse.go

@@ -16,7 +16,9 @@ package conf
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -35,25 +37,43 @@ type parser struct {
 
 	// Keys stack
 	keys []string
+
+	// The config file path, empty by default.
+	fp string
 }
 
 // Parse will return a map of keys to interface{}, although concrete types
 // underly them. The values supported are string, bool, int64, float64, DateTime.
 // Arrays and nested Maps are also supported.
 func Parse(data string) (map[string]interface{}, error) {
-	p, err := parse(data)
+	p, err := parse(data, "")
 	if err != nil {
 		return nil, err
 	}
 	return p.mapping, nil
 }
 
-func parse(data string) (p *parser, err error) {
+// ParseFile is a helper to open file, etc. and parse the contents.
+func ParseFile(fp string) (map[string]interface{}, error) {
+	data, err := ioutil.ReadFile(fp)
+	if err != nil {
+		return nil, fmt.Errorf("error opening config file: %v", err)
+	}
+
+	p, err := parse(string(data), filepath.Dir(fp))
+	if err != nil {
+		return nil, err
+	}
+	return p.mapping, nil
+}
+
+func parse(data, fp string) (p *parser, err error) {
 	p = &parser{
 		mapping: make(map[string]interface{}),
 		lx:      lex(data),
 		ctxs:    make([]interface{}, 0, 4),
 		keys:    make([]string, 0, 4),
+		fp:      fp,
 	}
 	p.pushContext(p.mapping)
 
@@ -152,7 +172,6 @@ func (p *parser) processItem(it item) error {
 		case "gb":
 			p.setValue(num * 1024 * 1024 * 1024)
 		}
-
 	case itemFloat:
 		num, err := strconv.ParseFloat(it.val, 64)
 		if err != nil {
@@ -193,6 +212,15 @@ func (p *parser) processItem(it item) error {
 			return fmt.Errorf("Variable reference for '%s' on line %d can not be found.",
 				it.val, it.line)
 		}
+	case itemInclude:
+		m, err := ParseFile(filepath.Join(p.fp, it.val))
+		if err != nil {
+			return fmt.Errorf("Error parsing include file '%s', %v.", it.val, err)
+		}
+		for k, v := range m {
+			p.pushKey(k)
+			p.setValue(v)
+		}
 	}
 
 	return nil

conf/parse_test.go

@@ -231,3 +231,34 @@ func TestSample5(t *testing.T) {
 	}
 	test(t, sample5, ex)
 }
+
+func TestIncludes(t *testing.T) {
+	ex := map[string]interface{}{
+		"listen": "127.0.0.1:4222",
+		"authorization": map[string]interface{}{
+			"ALICE_PASS": "$2a$10$UHR6GhotWhpLsKtVP0/i6.Nh9.fuY73cWjLoJjb2sKT8KISBcUW5q",
+			"BOB_PASS":   "$2a$11$dZM98SpGeI7dCFFGSpt.JObQcix8YHml4TBUZoge9R1uxnMIln5ly",
+			"users": []interface{}{
+				map[string]interface{}{
+					"user":     "alice",
+					"password": "$2a$10$UHR6GhotWhpLsKtVP0/i6.Nh9.fuY73cWjLoJjb2sKT8KISBcUW5q"},
+				map[string]interface{}{
+					"user":     "bob",
+					"password": "$2a$11$dZM98SpGeI7dCFFGSpt.JObQcix8YHml4TBUZoge9R1uxnMIln5ly"},
+			},
+			"timeout": float64(0.5),
+		},
+	}
+
+	m, err := ParseFile("simple.conf")
+	if err != nil {
+		t.Fatalf("Received err: %v\n", err)
+	}
+	if m == nil {
+		t.Fatal("Received nil map")
+	}
+
+	if !reflect.DeepEqual(m, ex) {
+		t.Fatalf("Not Equal:\nReceived: '%+v'\nExpected: '%+v'\n", m, ex)
+	}
+}

conf/simple.conf

@@ -0,0 +1,8 @@
+# Copyright 2016 Apcera Inc. All rights reserved.
+
+listen: 127.0.0.1:4222
+
+authorization {
+  include 'includes/users.conf' # Pull in from file
+  timeout: 0.5
+}

server/opts.go

@@ -130,12 +130,7 @@ func ProcessConfigFile(configFile string) (*Options, error) {
 		return opts, nil
 	}
 
-	data, err := ioutil.ReadFile(configFile)
+	m, err := conf.ParseFile(configFile)
-	if err != nil {
-		return nil, fmt.Errorf("error opening config file: %v", err)
-	}
-
-	m, err := conf.Parse(string(data))
 	if err != nil {
 		return nil, err
 	}

test/configs/authorization.conf

@@ -3,33 +3,8 @@
 listen: 127.0.0.1:2442
 
 authorization {
-  # Our role based permissions.
+  # Authorizations
-
+  include "auths.conf"
-  # Admin can do anything.
-  ADMIN = {
-    publish = ">"
-    subscribe = ">"
-  }
-
-  # Can do requests on req.foo or req.bar, and subscribe to anything
-  # that is a response, e.g. _INBOX.*
-  #
-  # Notice that authorization filters can be singletons or arrays.
-  REQUESTOR = {
-    publish = ["req.foo", "req.bar"]
-    subscribe = "_INBOX.*"
-  }
-
-  # Default permissions if none presented. e.g. Joe below.
-  DEFAULT_PERMISSIONS = {
-    publish = "SANDBOX.*"
-    subscribe = ["PUBLIC.>", "_INBOX.>"]
-  }
-
-  # This is to benchmark pub performance.
-  BENCH = {
-    publish = "a"
-  }
 
   # Just foo for testing
   PASS: $2a$10$UHR6GhotWhpLsKtVP0/i6.Nh9.fuY73cWjLoJjb2sKT8KISBcUW5q

test/user_authorization_test.go

@@ -87,5 +87,4 @@ func TestUserAuthorizationProto(t *testing.T) {
 	expectResult(t, c, permErrRe)
 
 	c.Close()
-
 }