diff --git a/logger/log.go b/logger/log.go index b77cf4de..9471cf62 100644 --- a/logger/log.go +++ b/logger/log.go @@ -90,6 +90,22 @@ func NewFileLogger(filename string, time, debug, trace, pid bool) *Logger { return l } +// NewTestLogger creates a logger with output directed to Stderr with a prefix. +// Useful for tracing in tests when multiple servers are in the same pid +func NewTestLogger(prefix string, time bool) *Logger { + flags := 0 + if time { + flags = log.LstdFlags | log.Lmicroseconds + } + l := &Logger{ + logger: log.New(os.Stderr, prefix, flags), + debug: true, + trace: true, + } + setColoredLabelFormats(l) + return l +} + // Close implements the io.Closer interface to clean up // resources in the server's logger implementation. // Caller must ensure threadsafety. diff --git a/server/accounts.go b/server/accounts.go new file mode 100644 index 00000000..5023f609 --- /dev/null +++ b/server/accounts.go @@ -0,0 +1,431 @@ +// Copyright 2018 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "sort" + "strings" + "sync" + "time" +) + +// For backwards compatibility, users who are not explicitly defined into an +// account will be grouped in the default global account. +const globalAccountName = "$G" + +// Route Map Entry - used for efficient interest graph propagation. +// TODO(dlc) - squeeze even more? +type rme struct { + qi int // used to index into key from map for optional queue name + n int32 // number of subscriptions directly matching, local subs only. +} + +// Accounts +type Account struct { + Name string + Nkey string + mu sync.RWMutex + sl *Sublist + clients int + rm map[string]*rme + imports importMap + exports exportMap + nae int + maxnae int + maxaettl time.Duration + pruning bool +} + +// Import stream mapping struct +type streamImport struct { + acc *Account + from string + prefix string +} + +// Import service mapping struct +type serviceImport struct { + acc *Account + from string + to string + ae bool + ts int64 +} + +// importMap tracks the imported streams and services. +type importMap struct { + streams map[string]*streamImport + services map[string]*serviceImport // TODO(dlc) sync.Map may be better. +} + +// exportMap tracks the exported streams and services. +type exportMap struct { + streams map[string]map[string]*Account + services map[string]map[string]*Account +} + +// NumClients returns active number of clients for this account. +func (a *Account) NumClients() int { + a.mu.RLock() + defer a.mu.RUnlock() + return a.clients +} + +// Returns how many subjects we would send across a route when first +// connected or expressing interest. Local client subs. +func (a *Account) RoutedSubs() int { + a.mu.RLock() + defer a.mu.RUnlock() + return len(a.rm) +} + +// Returns total number of Subscriptions for this account. +func (a *Account) TotalSubs() int { + a.mu.RLock() + defer a.mu.RUnlock() + return int(a.sl.Count()) +} + +// addClient keeps our accounting of active clients updated. +// Call in with client but just track total for now. +// Returns previous total. +func (a *Account) addClient(c *client) int { + a.mu.Lock() + n := a.clients + a.clients++ + a.mu.Unlock() + return n +} + +// removeClient keeps our accounting of active clients updated. +func (a *Account) removeClient(c *client) int { + a.mu.Lock() + n := a.clients + a.clients-- + a.mu.Unlock() + return n +} + +func (a *Account) AddServiceExport(subject string, accounts []*Account) error { + a.mu.Lock() + defer a.mu.Unlock() + if a == nil { + return ErrMissingAccount + } + if a.exports.services == nil { + a.exports.services = make(map[string]map[string]*Account) + } + ma := a.exports.services[subject] + if accounts != nil && ma == nil { + ma = make(map[string]*Account) + } + for _, a := range accounts { + ma[a.Name] = a + } + a.exports.services[subject] = ma + return nil +} + +// numServiceRoutes returns the number of service routes on this account. +func (a *Account) numServiceRoutes() int { + a.mu.RLock() + defer a.mu.RUnlock() + return len(a.imports.services) +} + +// This will add a route to an account to send published messages / requests +// to the destination account. From is the local subject to map, To is the +// subject that will appear on the destination account. Destination will need +// to have an import rule to allow access via addService. +func (a *Account) AddServiceImport(destination *Account, from, to string) error { + if destination == nil { + return ErrMissingAccount + } + // Empty means use from. + if to == "" { + to = from + } + if !IsValidLiteralSubject(from) || !IsValidLiteralSubject(to) { + return ErrInvalidSubject + } + // First check to see if the account has authorized us to route to the "to" subject. + if !destination.checkServiceImportAuthorized(a, to) { + return ErrServiceImportAuthorization + } + + return a.addImplicitServiceImport(destination, from, to, false) +} + +// removeServiceImport will remove the route by subject. +func (a *Account) removeServiceImport(subject string) { + a.mu.Lock() + si, ok := a.imports.services[subject] + if ok && si != nil && si.ae { + a.nae-- + } + delete(a.imports.services, subject) + a.mu.Unlock() +} + +// Return the number of AutoExpireResponseMaps for request/reply. These are mapped to the account that +// has the service import. +func (a *Account) numAutoExpireResponseMaps() int { + a.mu.RLock() + defer a.mu.RUnlock() + return a.nae +} + +// maxAutoExpireResponseMaps return the maximum number of +// auto expire response maps we will allow. +func (a *Account) MaxAutoExpireResponseMaps() int { + a.mu.RLock() + defer a.mu.RUnlock() + return a.maxnae +} + +// Set the max outstanding auto expire response maps. +func (a *Account) SetMaxAutoExpireResponseMaps(max int) { + a.mu.Lock() + defer a.mu.Unlock() + a.maxnae = max +} + +// expireTTL returns the ttl for response maps. +func (a *Account) AutoExpireTTL() time.Duration { + a.mu.RLock() + defer a.mu.RUnlock() + return a.maxaettl +} + +// Set the ttl for response maps. +func (a *Account) SetAutoExpireTTL(ttl time.Duration) { + a.mu.Lock() + defer a.mu.Unlock() + a.maxaettl = ttl +} + +// Return a list of the current autoExpireResponseMaps. +func (a *Account) autoExpireResponseMaps() []*serviceImport { + a.mu.RLock() + defer a.mu.RUnlock() + if len(a.imports.services) == 0 { + return nil + } + aesis := make([]*serviceImport, 0, len(a.imports.services)) + for _, si := range a.imports.services { + if si.ae { + aesis = append(aesis, si) + } + } + sort.Slice(aesis, func(i, j int) bool { + return aesis[i].ts < aesis[j].ts + }) + return aesis +} + +// Add a route to connect from an implicit route created for a response to a request. +// This does no checks and should be only called by the msg processing code. Use +// addServiceImport from above if responding to user input or config, etc. +func (a *Account) addImplicitServiceImport(destination *Account, from, to string, autoexpire bool) error { + a.mu.Lock() + if a.imports.services == nil { + a.imports.services = make(map[string]*serviceImport) + } + si := &serviceImport{destination, from, to, autoexpire, 0} + a.imports.services[from] = si + if autoexpire { + a.nae++ + si.ts = time.Now().Unix() + if a.nae > a.maxnae && !a.pruning { + a.pruning = true + go a.pruneAutoExpireResponseMaps() + } + } + a.mu.Unlock() + return nil +} + +// This will prune the list to below the threshold and remove all ttl'd maps. +func (a *Account) pruneAutoExpireResponseMaps() { + defer func() { + a.mu.Lock() + a.pruning = false + a.mu.Unlock() + }() + + a.mu.RLock() + ttl := int64(a.maxaettl/time.Second) + 1 + a.mu.RUnlock() + + for { + sis := a.autoExpireResponseMaps() + + // Check ttl items. + now := time.Now().Unix() + for i, si := range sis { + if now-si.ts >= ttl { + a.removeServiceImport(si.from) + } else { + sis = sis[i:] + break + } + } + + a.mu.RLock() + numOver := a.nae - a.maxnae + a.mu.RUnlock() + + if numOver <= 0 { + return + } else if numOver >= len(sis) { + numOver = len(sis) - 1 + } + // These are in sorted order, remove at least numOver + for _, si := range sis[:numOver] { + a.removeServiceImport(si.from) + } + } +} + +// addStreamImport will add in the stream import from a specific account. +func (a *Account) AddStreamImport(account *Account, from, prefix string) error { + if account == nil { + return ErrMissingAccount + } + + // First check to see if the account has authorized export of the subject. + if !account.checkStreamImportAuthorized(a, from) { + return ErrStreamImportAuthorization + } + + a.mu.Lock() + defer a.mu.Unlock() + if a.imports.streams == nil { + a.imports.streams = make(map[string]*streamImport) + } + if prefix != "" && prefix[len(prefix)-1] != btsep { + prefix = prefix + string(btsep) + } + // TODO(dlc) - collisions, etc. + a.imports.streams[from] = &streamImport{account, from, prefix} + return nil +} + +// Placeholder to denote public export. +var IsPublicExport = []*Account(nil) + +// addExport will add an export to the account. If accounts is nil +// it will signify a public export, meaning anyone can impoort. +func (a *Account) AddStreamExport(subject string, accounts []*Account) error { + a.mu.Lock() + defer a.mu.Unlock() + if a == nil { + return ErrMissingAccount + } + if a.exports.streams == nil { + a.exports.streams = make(map[string]map[string]*Account) + } + var ma map[string]*Account + for _, aa := range accounts { + if ma == nil { + ma = make(map[string]*Account, len(accounts)) + } + ma[aa.Name] = aa + } + a.exports.streams[subject] = ma + return nil +} + +// Check if another account is authorized to import from us. +func (a *Account) checkStreamImportAuthorized(account *Account, subject string) bool { + // Find the subject in the exports list. + a.mu.RLock() + defer a.mu.RUnlock() + + if a.exports.streams == nil || !IsValidSubject(subject) { + return false + } + + // Check direct match of subject first + am, ok := a.exports.streams[subject] + if ok { + // if am is nil that denotes a public export + if am == nil { + return true + } + // If we have a matching account we are authorized + _, ok := am[account.Name] + return ok + } + // ok if we are here we did not match directly so we need to test each one. + // The import subject arg has to take precedence, meaning the export + // has to be a true subset of the import claim. We already checked for + // exact matches above. + tokens := strings.Split(subject, tsep) + for subj, am := range a.exports.streams { + if isSubsetMatch(tokens, subj) { + if am == nil { + return true + } + _, ok := am[account.Name] + return ok + } + } + return false +} + +// Returns true if `a` and `b` stream imports are the same. Note that the +// check is done with the account's name, not the pointer. This is used +// during config reload where we are comparing current and new config +// in which pointers are different. +// No lock is acquired in this function, so it is assumed that the +// import maps are not changed while this executes. +func (a *Account) checkStreamImportsEqual(b *Account) bool { + if len(a.imports.streams) != len(b.imports.streams) { + return false + } + for subj, aim := range a.imports.streams { + bim := b.imports.streams[subj] + if bim == nil { + return false + } + if aim.acc.Name != bim.acc.Name || aim.from != bim.from || aim.prefix != bim.prefix { + return false + } + } + return true +} + +// Check if another account is authorized to route requests to this service. +func (a *Account) checkServiceImportAuthorized(account *Account, subject string) bool { + // Find the subject in the services list. + a.mu.RLock() + defer a.mu.RUnlock() + + if a.exports.services == nil || !IsValidLiteralSubject(subject) { + return false + } + // These are always literal subjects so just lookup. + am, ok := a.exports.services[subject] + if !ok { + return false + } + // Check to see if we are public or if we need to search for the account. + if am == nil { + return true + } + // Check that we allow this account. + _, ok = am[account.Name] + return ok +} diff --git a/server/accounts_test.go b/server/accounts_test.go index 29e116b3..4d68f2cd 100644 --- a/server/accounts_test.go +++ b/server/accounts_test.go @@ -30,11 +30,11 @@ func simpleAccountServer(t *testing.T) (*Server, *Account, *Account) { s := New(&opts) // Now create two accounts. - f, err := s.RegisterAccount("foo") + f, err := s.RegisterAccount("$foo") if err != nil { t.Fatalf("Error creating account 'foo': %v", err) } - b, err := s.RegisterAccount("bar") + b, err := s.RegisterAccount("$bar") if err != nil { t.Fatalf("Error creating account 'bar': %v", err) } @@ -43,7 +43,7 @@ func simpleAccountServer(t *testing.T) (*Server, *Account, *Account) { func TestRegisterDuplicateAccounts(t *testing.T) { s, _, _ := simpleAccountServer(t) - if _, err := s.RegisterAccount("foo"); err == nil { + if _, err := s.RegisterAccount("$foo"); err == nil { t.Fatal("Expected an error registering 'foo' twice") } } @@ -153,6 +153,91 @@ func TestNewAccountsFromClients(t *testing.T) { } } +func TestActiveAccounts(t *testing.T) { + opts := defaultServerOptions + opts.AllowNewAccounts = true + opts.Cluster.Port = 22 + + s := New(&opts) + + if s.activeAccounts != 0 { + t.Fatalf("Expected no active accounts, got %d", s.activeAccounts) + } + + addClientWithAccount := func(accName string) *client { + t.Helper() + c, _, _ := newClientForServer(s) + connectOp := fmt.Sprintf("CONNECT {\"account\":\"%s\"}\r\n", accName) + err := c.parse([]byte(connectOp)) + if err != nil { + t.Fatalf("Received an error trying to connect: %v", err) + } + return c + } + + // Now add some clients. + cf1 := addClientWithAccount("foo") + if s.activeAccounts != 1 { + t.Fatalf("Expected active accounts to be 1, got %d", s.activeAccounts) + } + // Adding in same one should not change total. + cf2 := addClientWithAccount("foo") + if s.activeAccounts != 1 { + t.Fatalf("Expected active accounts to be 1, got %d", s.activeAccounts) + } + // Add in new one. + cb1 := addClientWithAccount("bar") + if s.activeAccounts != 2 { + t.Fatalf("Expected active accounts to be 2, got %d", s.activeAccounts) + } + + // Make sure the Accounts track clients. + foo := s.LookupAccount("foo") + bar := s.LookupAccount("bar") + if foo == nil || bar == nil { + t.Fatalf("Error looking up accounts") + } + if nc := foo.NumClients(); nc != 2 { + t.Fatalf("Expected account foo to have 2 clients, got %d", nc) + } + if nc := bar.NumClients(); nc != 1 { + t.Fatalf("Expected account bar to have 1 client, got %d", nc) + } + + waitTilActiveCount := func(n int) { + t.Helper() + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if active := s.activeAccounts; active != n { + return fmt.Errorf("Number of active accounts is %d", active) + } + return nil + }) + } + + // Test Removal + cb1.closeConnection(ClientClosed) + waitTilActiveCount(1) + + if nc := bar.NumClients(); nc != 0 { + t.Fatalf("Expected account bar to have 0 clients, got %d", nc) + } + + // This should not change the count. + cf1.closeConnection(ClientClosed) + waitTilActiveCount(1) + + if nc := foo.NumClients(); nc != 1 { + t.Fatalf("Expected account foo to have 1 client, got %d", nc) + } + + cf2.closeConnection(ClientClosed) + waitTilActiveCount(0) + + if nc := foo.NumClients(); nc != 0 { + t.Fatalf("Expected account bar to have 0 clients, got %d", nc) + } +} + // Clients can ask that the account be forced to be new. If it exists this is an error. func TestNewAccountRequireNew(t *testing.T) { // This has foo and bar accounts already. @@ -448,12 +533,12 @@ func TestImportAuthorized(t *testing.T) { checkBool(foo.checkStreamImportAuthorized(bar, "foo.*"), false, t) checkBool(foo.checkStreamImportAuthorized(bar, "foo.>"), false, t) - foo.addStreamExport("foo", isPublicExport) + foo.AddStreamExport("foo", IsPublicExport) checkBool(foo.checkStreamImportAuthorized(bar, "foo"), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "bar"), false, t) checkBool(foo.checkStreamImportAuthorized(bar, "*"), false, t) - foo.addStreamExport("*", []*Account{bar}) + foo.AddStreamExport("*", []*Account{bar}) checkBool(foo.checkStreamImportAuthorized(bar, "foo"), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "bar"), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "baz"), true, t) @@ -466,7 +551,7 @@ func TestImportAuthorized(t *testing.T) { // Reset and test '>' public export _, foo, bar = simpleAccountServer(t) - foo.addStreamExport(">", nil) + foo.AddStreamExport(">", nil) // Everything should work. checkBool(foo.checkStreamImportAuthorized(bar, "foo"), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "bar"), true, t) @@ -480,7 +565,7 @@ func TestImportAuthorized(t *testing.T) { // Reset and test pwc and fwc s, foo, bar := simpleAccountServer(t) - foo.addStreamExport("foo.*.baz.>", []*Account{bar}) + foo.AddStreamExport("foo.*.baz.>", []*Account{bar}) checkBool(foo.checkStreamImportAuthorized(bar, "foo.bar.baz.1"), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "foo.bar.baz.*"), true, t) checkBool(foo.checkStreamImportAuthorized(bar, "foo.*.baz.1.1"), true, t) @@ -516,16 +601,16 @@ func TestSimpleMapping(t *testing.T) { } // Test first that trying to import with no matching export permission returns an error. - if err := cbar.acc.addStreamImport(fooAcc, "foo", "import"); err != ErrStreamImportAuthorization { + if err := cbar.acc.AddStreamImport(fooAcc, "foo", "import"); err != ErrStreamImportAuthorization { t.Fatalf("Expected error of ErrAccountImportAuthorization but got %v", err) } // Now map the subject space between foo and bar. // Need to do export first. - if err := cfoo.acc.addStreamExport("foo", nil); err != nil { // Public with no accounts defined. + if err := cfoo.acc.AddStreamExport("foo", nil); err != nil { // Public with no accounts defined. t.Fatalf("Error adding account export to client foo: %v", err) } - if err := cbar.acc.addStreamImport(fooAcc, "foo", "import"); err != nil { + if err := cbar.acc.AddStreamImport(fooAcc, "foo", "import"); err != nil { t.Fatalf("Error adding account import to client bar: %v", err) } @@ -608,10 +693,10 @@ func TestNoPrefixWildcardMapping(t *testing.T) { t.Fatalf("Error registering client with 'bar' account: %v", err) } - if err := cfoo.acc.addStreamExport(">", []*Account{barAcc}); err != nil { + if err := cfoo.acc.AddStreamExport(">", []*Account{barAcc}); err != nil { t.Fatalf("Error adding stream export to client foo: %v", err) } - if err := cbar.acc.addStreamImport(fooAcc, "*", ""); err != nil { + if err := cbar.acc.AddStreamImport(fooAcc, "*", ""); err != nil { t.Fatalf("Error adding stream import to client bar: %v", err) } @@ -661,11 +746,11 @@ func TestPrefixWildcardMapping(t *testing.T) { t.Fatalf("Error registering client with 'bar' account: %v", err) } - if err := cfoo.acc.addStreamExport(">", []*Account{barAcc}); err != nil { + if err := cfoo.acc.AddStreamExport(">", []*Account{barAcc}); err != nil { t.Fatalf("Error adding stream export to client foo: %v", err) } // Checking that trailing '.' is accepted, tested that it is auto added above. - if err := cbar.acc.addStreamImport(fooAcc, "*", "pub.imports."); err != nil { + if err := cbar.acc.AddStreamImport(fooAcc, "*", "pub.imports."); err != nil { t.Fatalf("Error adding stream import to client bar: %v", err) } @@ -715,10 +800,10 @@ func TestPrefixWildcardMappingWithLiteralSub(t *testing.T) { t.Fatalf("Error registering client with 'bar' account: %v", err) } - if err := cfoo.acc.addStreamExport(">", []*Account{barAcc}); err != nil { + if err := fooAcc.AddStreamExport(">", []*Account{barAcc}); err != nil { t.Fatalf("Error adding stream export to client foo: %v", err) } - if err := cbar.acc.addStreamImport(fooAcc, "*", "pub.imports."); err != nil { + if err := barAcc.AddStreamImport(fooAcc, "*", "pub.imports."); err != nil { t.Fatalf("Error adding stream import to client bar: %v", err) } @@ -769,23 +854,23 @@ func TestCrossAccountRequestReply(t *testing.T) { } // Add in the service export for the requests. Make it public. - if err := cfoo.acc.addServiceExport("test.request", nil); err != nil { + if err := cfoo.acc.AddServiceExport("test.request", nil); err != nil { t.Fatalf("Error adding account service export to client foo: %v", err) } // Test addServiceImport to make sure it requires accounts, and literalsubjects for both from and to subjects. - if err := cbar.acc.addServiceImport(nil, "foo", "test.request"); err != ErrMissingAccount { + if err := cbar.acc.AddServiceImport(nil, "foo", "test.request"); err != ErrMissingAccount { t.Fatalf("Expected ErrMissingAccount but received %v.", err) } - if err := cbar.acc.addServiceImport(fooAcc, "*", "test.request"); err != ErrInvalidSubject { + if err := cbar.acc.AddServiceImport(fooAcc, "*", "test.request"); err != ErrInvalidSubject { t.Fatalf("Expected ErrInvalidSubject but received %v.", err) } - if err := cbar.acc.addServiceImport(fooAcc, "foo", "test..request."); err != ErrInvalidSubject { + if err := cbar.acc.AddServiceImport(fooAcc, "foo", "test..request."); err != ErrInvalidSubject { t.Fatalf("Expected ErrInvalidSubject but received %v.", err) } // Now add in the Route for request to be routed to the foo account. - if err := cbar.acc.addServiceImport(fooAcc, "foo", "test.request"); err != nil { + if err := cbar.acc.AddServiceImport(fooAcc, "foo", "test.request"); err != nil { t.Fatalf("Error adding account service import to client bar: %v", err) } @@ -814,8 +899,8 @@ func TestCrossAccountRequestReply(t *testing.T) { t.Fatalf("Did not get correct sid: '%s'", matches[SID_INDEX]) } // Make sure this looks like _INBOX - if !strings.HasPrefix(matches[REPLY_INDEX], "_INBOX.") { - t.Fatalf("Expected an _INBOX.* like reply, got '%s'", matches[REPLY_INDEX]) + if !strings.HasPrefix(matches[REPLY_INDEX], "_R_.") { + t.Fatalf("Expected an _R_.* like reply, got '%s'", matches[REPLY_INDEX]) } checkPayload(crFoo, []byte("help\r\n"), t) @@ -854,9 +939,18 @@ func TestCrossAccountRequestReplyResponseMaps(t *testing.T) { s, fooAcc, barAcc := simpleAccountServer(t) defer s.Shutdown() + // Make sure they have the correct defaults + if max := barAcc.MaxAutoExpireResponseMaps(); max != DEFAULT_MAX_ACCOUNT_AE_RESPONSE_MAPS { + t.Fatalf("Expected %d for max default, but got %d", DEFAULT_MAX_ACCOUNT_AE_RESPONSE_MAPS, max) + } + + if ttl := barAcc.AutoExpireTTL(); ttl != DEFAULT_TTL_AE_RESPONSE_MAP { + t.Fatalf("Expected %v for the ttl default, got %v", DEFAULT_TTL_AE_RESPONSE_MAP, ttl) + } + ttl := 500 * time.Millisecond - barAcc.setMaxAutoExpireResponseMaps(5) - barAcc.setMaxAutoExpireTTL(ttl) + barAcc.SetMaxAutoExpireResponseMaps(5) + barAcc.SetAutoExpireTTL(ttl) cfoo, _, _ := newClientForServer(s) defer cfoo.nc.Close() @@ -864,10 +958,10 @@ func TestCrossAccountRequestReplyResponseMaps(t *testing.T) { t.Fatalf("Error registering client with 'foo' account: %v", err) } - if err := barAcc.addServiceExport("test.request", nil); err != nil { + if err := barAcc.AddServiceExport("test.request", nil); err != nil { t.Fatalf("Error adding account service export: %v", err) } - if err := fooAcc.addServiceImport(barAcc, "foo", "test.request"); err != nil { + if err := fooAcc.AddServiceImport(barAcc, "foo", "test.request"); err != nil { t.Fatalf("Error adding account service import: %v", err) } @@ -875,7 +969,7 @@ func TestCrossAccountRequestReplyResponseMaps(t *testing.T) { cfoo.parseAndFlush([]byte("PUB foo bar 4\r\nhelp\r\n")) } - // We should expire because max. + // We should expire because of max. checkFor(t, time.Second, 10*time.Millisecond, func() error { if nae := barAcc.numAutoExpireResponseMaps(); nae != 5 { return fmt.Errorf("Number of responsemaps is %d", nae) @@ -886,7 +980,7 @@ func TestCrossAccountRequestReplyResponseMaps(t *testing.T) { // Wait for the ttl to expire. time.Sleep(2 * ttl) - // Now run prune and make sure we collect the timedout ones. + // Now run prune and make sure we collect the timed-out ones. barAcc.pruneAutoExpireResponseMaps() // We should expire because ttl. @@ -939,10 +1033,6 @@ func TestAccountMapsUsers(t *testing.T) { if c.acc != synadia { t.Fatalf("Expected the client's account to match 'synadia', got %v", c.acc) } - // Also test client sublist. - if c.sl != synadia.sl { - t.Fatalf("Expected the client's sublist to match 'synadia' account") - } c, _, _ = newClientForServer(s) connectOp = []byte("CONNECT {\"user\":\"ivan\",\"pass\":\"bar\"}\r\n") @@ -950,10 +1040,6 @@ func TestAccountMapsUsers(t *testing.T) { if c.acc != nats { t.Fatalf("Expected the client's account to match 'nats', got %v", c.acc) } - // Also test client sublist. - if c.sl != nats.sl { - t.Fatalf("Expected the client's sublist to match 'nats' account") - } // Now test nkeys as well. kp, _ := nkeys.FromSeed(seed1) @@ -985,10 +1071,6 @@ func TestAccountMapsUsers(t *testing.T) { if c.acc != synadia { t.Fatalf("Expected the nkey client's account to match 'synadia', got %v", c.acc) } - // Also test client sublist. - if c.sl != synadia.sl { - t.Fatalf("Expected the client's sublist to match 'synadia' account") - } // Now nats account nkey user. kp, _ = nkeys.FromSeed(seed2) @@ -1019,10 +1101,6 @@ func TestAccountMapsUsers(t *testing.T) { if c.acc != nats { t.Fatalf("Expected the nkey client's account to match 'nats', got %v", c.acc) } - // Also test client sublist. - if c.sl != nats.sl { - t.Fatalf("Expected the client's sublist to match 'nats' account") - } } func TestAccountGlobalDefault(t *testing.T) { @@ -1049,29 +1127,29 @@ func TestAccountGlobalDefault(t *testing.T) { func TestAccountCheckStreamImportsEqual(t *testing.T) { // Create bare accounts for this test fooAcc := &Account{Name: "foo"} - if err := fooAcc.addStreamExport(">", nil); err != nil { + if err := fooAcc.AddStreamExport(">", nil); err != nil { t.Fatalf("Error adding stream export: %v", err) } barAcc := &Account{Name: "bar"} - if err := barAcc.addStreamImport(fooAcc, "foo", "myPrefix"); err != nil { + if err := barAcc.AddStreamImport(fooAcc, "foo", "myPrefix"); err != nil { t.Fatalf("Error adding stream import: %v", err) } bazAcc := &Account{Name: "baz"} - if err := bazAcc.addStreamImport(fooAcc, "foo", "myPrefix"); err != nil { + if err := bazAcc.AddStreamImport(fooAcc, "foo", "myPrefix"); err != nil { t.Fatalf("Error adding stream import: %v", err) } if !barAcc.checkStreamImportsEqual(bazAcc) { t.Fatal("Expected stream imports to be the same") } - if err := bazAcc.addStreamImport(fooAcc, "foo.>", ""); err != nil { + if err := bazAcc.AddStreamImport(fooAcc, "foo.>", ""); err != nil { t.Fatalf("Error adding stream import: %v", err) } if barAcc.checkStreamImportsEqual(bazAcc) { t.Fatal("Expected stream imports to be different") } - if err := barAcc.addStreamImport(fooAcc, "foo.>", ""); err != nil { + if err := barAcc.AddStreamImport(fooAcc, "foo.>", ""); err != nil { t.Fatalf("Error adding stream import: %v", err) } if !barAcc.checkStreamImportsEqual(bazAcc) { @@ -1081,14 +1159,14 @@ func TestAccountCheckStreamImportsEqual(t *testing.T) { // Create another account that is named "foo". We want to make sure // that the comparison still works (based on account name, not pointer) newFooAcc := &Account{Name: "foo"} - if err := newFooAcc.addStreamExport(">", nil); err != nil { + if err := newFooAcc.AddStreamExport(">", nil); err != nil { t.Fatalf("Error adding stream export: %v", err) } batAcc := &Account{Name: "bat"} - if err := batAcc.addStreamImport(newFooAcc, "foo", "myPrefix"); err != nil { + if err := batAcc.AddStreamImport(newFooAcc, "foo", "myPrefix"); err != nil { t.Fatalf("Error adding stream import: %v", err) } - if err := batAcc.addStreamImport(newFooAcc, "foo.>", ""); err != nil { + if err := batAcc.AddStreamImport(newFooAcc, "foo.>", ""); err != nil { t.Fatalf("Error adding stream import: %v", err) } if !batAcc.checkStreamImportsEqual(barAcc) { @@ -1100,15 +1178,15 @@ func TestAccountCheckStreamImportsEqual(t *testing.T) { // Test with account with different "from" expAcc := &Account{Name: "new_acc"} - if err := expAcc.addStreamExport(">", nil); err != nil { + if err := expAcc.AddStreamExport(">", nil); err != nil { t.Fatalf("Error adding stream export: %v", err) } aAcc := &Account{Name: "a"} - if err := aAcc.addStreamImport(expAcc, "bar", ""); err != nil { + if err := aAcc.AddStreamImport(expAcc, "bar", ""); err != nil { t.Fatalf("Error adding stream import: %v", err) } bAcc := &Account{Name: "b"} - if err := bAcc.addStreamImport(expAcc, "baz", ""); err != nil { + if err := bAcc.AddStreamImport(expAcc, "baz", ""); err != nil { t.Fatalf("Error adding stream import: %v", err) } if aAcc.checkStreamImportsEqual(bAcc) { @@ -1117,11 +1195,11 @@ func TestAccountCheckStreamImportsEqual(t *testing.T) { // Test with account with different "prefix" aAcc = &Account{Name: "a"} - if err := aAcc.addStreamImport(expAcc, "bar", "prefix"); err != nil { + if err := aAcc.AddStreamImport(expAcc, "bar", "prefix"); err != nil { t.Fatalf("Error adding stream import: %v", err) } bAcc = &Account{Name: "b"} - if err := bAcc.addStreamImport(expAcc, "bar", "diff_prefix"); err != nil { + if err := bAcc.AddStreamImport(expAcc, "bar", "diff_prefix"); err != nil { t.Fatalf("Error adding stream import: %v", err) } if aAcc.checkStreamImportsEqual(bAcc) { @@ -1130,11 +1208,11 @@ func TestAccountCheckStreamImportsEqual(t *testing.T) { // Test with account with different "name" expAcc = &Account{Name: "diff_name"} - if err := expAcc.addStreamExport(">", nil); err != nil { + if err := expAcc.AddStreamExport(">", nil); err != nil { t.Fatalf("Error adding stream export: %v", err) } bAcc = &Account{Name: "b"} - if err := bAcc.addStreamImport(expAcc, "bar", "prefix"); err != nil { + if err := bAcc.AddStreamImport(expAcc, "bar", "prefix"); err != nil { t.Fatalf("Error adding stream import: %v", err) } if aAcc.checkStreamImportsEqual(bAcc) { diff --git a/server/auth.go b/server/auth.go index 88f4e5ba..b297ae98 100644 --- a/server/auth.go +++ b/server/auth.go @@ -16,10 +16,7 @@ package server import ( "crypto/tls" "encoding/base64" - "sort" "strings" - "sync" - "time" "github.com/nats-io/nkeys" "golang.org/x/crypto/bcrypt" @@ -41,350 +38,6 @@ type ClientAuthentication interface { RegisterUser(*User) } -// For backwards compatibility, users who are not explicitly defined into an -// account will be grouped in the default global account. -const globalAccountName = "$G" - -// Accounts -type Account struct { - Name string - Nkey string - mu sync.RWMutex - sl *Sublist - imports importMap - exports exportMap - nae int - maxnae int - maxaettl time.Duration - pruning bool -} - -// Import stream mapping struct -type streamImport struct { - acc *Account - from string - prefix string -} - -// Import service mapping struct -type serviceImport struct { - acc *Account - from string - to string - ae bool - ts int64 -} - -// importMap tracks the imported streams and services. -type importMap struct { - streams map[string]*streamImport - services map[string]*serviceImport // TODO(dlc) sync.Map may be better. -} - -// exportMap tracks the exported streams and services. -type exportMap struct { - streams map[string]map[string]*Account - services map[string]map[string]*Account -} - -func (a *Account) addServiceExport(subject string, accounts []*Account) error { - a.mu.Lock() - defer a.mu.Unlock() - if a == nil { - return ErrMissingAccount - } - if a.exports.services == nil { - a.exports.services = make(map[string]map[string]*Account) - } - ma := a.exports.services[subject] - if accounts != nil && ma == nil { - ma = make(map[string]*Account) - } - for _, a := range accounts { - ma[a.Name] = a - } - a.exports.services[subject] = ma - return nil -} - -// numServiceRoutes returns the number of service routes on this account. -func (a *Account) numServiceRoutes() int { - a.mu.RLock() - defer a.mu.RUnlock() - return len(a.imports.services) -} - -// This will add a route to an account to send published messages / requests -// to the destination account. From is the local subject to map, To is the -// subject that will appear on the destination account. Destination will need -// to have an import rule to allow access via addService. -func (a *Account) addServiceImport(destination *Account, from, to string) error { - if destination == nil { - return ErrMissingAccount - } - // Empty means use from. - if to == "" { - to = from - } - if !IsValidLiteralSubject(from) || !IsValidLiteralSubject(to) { - return ErrInvalidSubject - } - // First check to see if the account has authorized us to route to the "to" subject. - if !destination.checkServiceImportAuthorized(a, to) { - return ErrServiceImportAuthorization - } - - return a.addImplicitServiceImport(destination, from, to, false) -} - -// removeServiceImport will remove the route by subject. -func (a *Account) removeServiceImport(subject string) { - a.mu.Lock() - si, ok := a.imports.services[subject] - if ok && si != nil && si.ae { - a.nae-- - } - delete(a.imports.services, subject) - a.mu.Unlock() -} - -// Return the number of AutoExpireResponseMaps for request/reply. These are mapped to the account that -func (a *Account) numAutoExpireResponseMaps() int { - a.mu.RLock() - defer a.mu.RUnlock() - return a.nae -} - -// Set the max outstanding auto expire response maps. -func (a *Account) setMaxAutoExpireResponseMaps(max int) { - a.mu.Lock() - defer a.mu.Unlock() - a.maxnae = max -} - -// Set the max ttl for response maps. -func (a *Account) setMaxAutoExpireTTL(ttl time.Duration) { - a.mu.Lock() - defer a.mu.Unlock() - a.maxaettl = ttl -} - -// Return a list of the current autoExpireResponseMaps. -func (a *Account) autoExpireResponseMaps() []*serviceImport { - a.mu.RLock() - defer a.mu.RUnlock() - if len(a.imports.services) == 0 { - return nil - } - aesis := make([]*serviceImport, 0, len(a.imports.services)) - for _, si := range a.imports.services { - if si.ae { - aesis = append(aesis, si) - } - } - sort.Slice(aesis, func(i, j int) bool { - return aesis[i].ts < aesis[j].ts - }) - return aesis -} - -// Add a route to connect from an implicit route created for a response to a request. -// This does no checks and should be only called by the msg processing code. Use addRoute -// above if responding to user input or config, etc. -func (a *Account) addImplicitServiceImport(destination *Account, from, to string, autoexpire bool) error { - a.mu.Lock() - if a.imports.services == nil { - a.imports.services = make(map[string]*serviceImport) - } - si := &serviceImport{destination, from, to, autoexpire, 0} - a.imports.services[from] = si - if autoexpire { - a.nae++ - si.ts = time.Now().Unix() - if a.nae > a.maxnae && !a.pruning { - a.pruning = true - go a.pruneAutoExpireResponseMaps() - } - } - a.mu.Unlock() - return nil -} - -// This will prune the list to below the threshold and remove all ttl'd maps. -func (a *Account) pruneAutoExpireResponseMaps() { - defer func() { - a.mu.Lock() - a.pruning = false - a.mu.Unlock() - }() - - a.mu.RLock() - ttl := int64(a.maxaettl/time.Second) + 1 - a.mu.RUnlock() - - for { - sis := a.autoExpireResponseMaps() - - // Check ttl items. - now := time.Now().Unix() - for i, si := range sis { - if now-si.ts >= ttl { - a.removeServiceImport(si.from) - } else { - sis = sis[i:] - break - } - } - - a.mu.RLock() - numOver := a.nae - a.maxnae - a.mu.RUnlock() - - if numOver <= 0 { - return - } else if numOver >= len(sis) { - numOver = len(sis) - 1 - } - // These are in sorted order, remove at least numOver - for _, si := range sis[:numOver] { - a.removeServiceImport(si.from) - } - } -} - -// addStreamImport will add in the stream import from a specific account. -func (a *Account) addStreamImport(account *Account, from, prefix string) error { - if account == nil { - return ErrMissingAccount - } - - // First check to see if the account has authorized export of the subject. - if !account.checkStreamImportAuthorized(a, from) { - return ErrStreamImportAuthorization - } - - a.mu.Lock() - defer a.mu.Unlock() - if a.imports.streams == nil { - a.imports.streams = make(map[string]*streamImport) - } - if prefix != "" && prefix[len(prefix)-1] != btsep { - prefix = prefix + string(btsep) - } - // TODO(dlc) - collisions, etc. - a.imports.streams[from] = &streamImport{account, from, prefix} - return nil -} - -// Placeholder to denote public export. -var isPublicExport = []*Account(nil) - -// addExport will add an export to the account. If accounts is nil -// it will signify a public export, meaning anyone can impoort. -func (a *Account) addStreamExport(subject string, accounts []*Account) error { - a.mu.Lock() - defer a.mu.Unlock() - if a == nil { - return ErrMissingAccount - } - if a.exports.streams == nil { - a.exports.streams = make(map[string]map[string]*Account) - } - var ma map[string]*Account - for _, aa := range accounts { - if ma == nil { - ma = make(map[string]*Account, len(accounts)) - } - ma[aa.Name] = aa - } - a.exports.streams[subject] = ma - return nil -} - -// Check if another account is authorized to import from us. -func (a *Account) checkStreamImportAuthorized(account *Account, subject string) bool { - // Find the subject in the exports list. - a.mu.RLock() - defer a.mu.RUnlock() - - if a.exports.streams == nil || !IsValidSubject(subject) { - return false - } - - // Check direct match of subject first - am, ok := a.exports.streams[subject] - if ok { - // if am is nil that denotes a public export - if am == nil { - return true - } - // If we have a matching account we are authorized - _, ok := am[account.Name] - return ok - } - // ok if we are here we did not match directly so we need to test each one. - // The import subject arg has to take precedence, meaning the export - // has to be a true subset of the import claim. We already checked for - // exact matches above. - - tokens := strings.Split(subject, tsep) - for subj, am := range a.exports.streams { - if isSubsetMatch(tokens, subj) { - if am == nil { - return true - } - _, ok := am[account.Name] - return ok - } - } - return false -} - -// Returns true if `a` and `b` stream imports are the same. Note that the -// check is done with the account's name, not the pointer. This is used -// during config reload where we are comparing current and new config -// in which pointers are different. -// No lock is acquired in this function, so it is assumed that the -// import maps are not changed while this executes. -func (a *Account) checkStreamImportsEqual(b *Account) bool { - if len(a.imports.streams) != len(b.imports.streams) { - return false - } - for subj, aim := range a.imports.streams { - bim := b.imports.streams[subj] - if bim == nil { - return false - } - if aim.acc.Name != bim.acc.Name || aim.from != bim.from || aim.prefix != bim.prefix { - return false - } - } - return true -} - -// Check if another account is authorized to route requests to this service. -func (a *Account) checkServiceImportAuthorized(account *Account, subject string) bool { - // Find the subject in the services list. - a.mu.RLock() - defer a.mu.RUnlock() - - if a.exports.services == nil || !IsValidLiteralSubject(subject) { - return false - } - // These are always literal subjects so just lookup. - am, ok := a.exports.services[subject] - if !ok { - return false - } - // Check to see if we are public or if we need to search for the account. - if am == nil { - return true - } - // Check that we allow this account. - _, ok = am[account.Name] - return ok -} - // Nkey is for multiple nkey based users type NkeyUser struct { Nkey string `json:"user"` diff --git a/server/client.go b/server/client.go index b0fa2ce5..57f38ebf 100644 --- a/server/client.go +++ b/server/client.go @@ -51,8 +51,8 @@ func init() { const ( // Scratch buffer size for the processMsg() calls. - msgScratchSize = 512 - msgHeadProto = "MSG " + msgScratchSize = 1024 + msgHeadProto = "RMSG " msgHeadProtoLen = len(msgHeadProto) ) @@ -146,7 +146,6 @@ type client struct { out outbound srv *Server acc *Account - sl *Sublist subs map[string]*subscription perms *permissions in readCache @@ -240,6 +239,8 @@ func (c *client) GetTLSConnectionState() *tls.ConnectionState { // This is the main subscription struct that indicates // interest in published messages. +// FIXME(dlc) - This is getting bloated for normal subs, need +// to optionally have an opts section for non-normal stuff. type subscription struct { client *client im *streamImport // This is for import stream support. @@ -249,6 +250,7 @@ type subscription struct { sid []byte nm int64 max int64 + qw int32 } type clientOpts struct { @@ -299,9 +301,10 @@ func (c *client) initClient() { c.trace = (atomic.LoadInt32(&c.srv.logging.trace) != 0) // This is a scratch buffer used for processMsg() - // The msg header starts with "MSG ", - // in bytes that is [77 83 71 32]. - c.msgb = [msgScratchSize]byte{77, 83, 71, 32} + // The msg header starts with "RMSG ", which can be used + // for both local and routes. + // in bytes that is [82 77 83 71 32]. + c.msgb = [msgScratchSize]byte{82, 77, 83, 71, 32} // This is to track pending clients that have data to be flushed // after we process inbound msgs from our own connection. @@ -328,9 +331,22 @@ func (c *client) registerWithAccount(acc *Account) error { if acc == nil || acc.sl == nil { return ErrBadAccount } + // If we were previously register, usually to $G, do accounting here to remove. + if c.acc != nil { + if prev := c.acc.removeClient(c); prev == 1 && c.srv != nil { + c.srv.mu.Lock() + c.srv.activeAccounts-- + c.srv.mu.Unlock() + } + } + // Add in new one. + if prev := acc.addClient(c); prev == 0 && c.srv != nil { + c.srv.mu.Lock() + c.srv.activeAccounts++ + c.srv.mu.Unlock() + } c.mu.Lock() c.acc = acc - c.sl = acc.sl c.mu.Unlock() return nil } @@ -339,15 +355,18 @@ func (c *client) registerWithAccount(acc *Account) error { // with the authenticated user. This is used to map // any permissions into the client and setup accounts. func (c *client) RegisterUser(user *User) { - c.mu.Lock() - defer c.mu.Unlock() - // Register with proper account and sublist. if user.Account != nil { - c.acc = user.Account - c.sl = c.acc.sl + if err := c.registerWithAccount(user.Account); err != nil { + c.Errorf("Problem registering with account [%s]", user.Account.Name) + c.sendErr("Failed Account Registration") + return + } } + c.mu.Lock() + defer c.mu.Unlock() + // Assign permissions. if user.Permissions == nil { // Reset perms to nil in case client previously had them. @@ -368,7 +387,6 @@ func (c *client) RegisterNkeyUser(user *NkeyUser) { // Register with proper account and sublist. if user.Account != nil { c.acc = user.Account - c.sl = c.acc.sl } // Assign permissions. @@ -707,15 +725,15 @@ func (c *client) traceMsg(msg []byte) { return } // FIXME(dlc), allow limits to printable payload - c.Tracef("->> MSG_PAYLOAD: [%s]", string(msg[:len(msg)-LEN_CR_LF])) + c.Tracef("<<- MSG_PAYLOAD: [%s]", string(msg[:len(msg)-LEN_CR_LF])) } func (c *client) traceInOp(op string, arg []byte) { - c.traceOp("->> %s", op, arg) + c.traceOp("<<- %s", op, arg) } func (c *client) traceOutOp(op string, arg []byte) { - c.traceOp("<<- %s", op, arg) + c.traceOp("->> %s", op, arg) } func (c *client) traceOp(format, op string, arg []byte) { @@ -852,7 +870,7 @@ func (c *client) processConnect(arg []byte) error { if account != "" { var acc *Account var wasNew bool - if !srv.newAccountsAllowed() { + if !srv.NewAccountsAllowed() { acc = srv.LookupAccount(account) if acc == nil { c.Errorf(ErrMissingAccount.Error()) @@ -875,10 +893,14 @@ func (c *client) processConnect(arg []byte) error { // If we are here we can register ourselves with the new account. if err := c.registerWithAccount(acc); err != nil { c.Errorf("Problem registering with account [%s]", account) - c.sendErr("Account Failed Registration") + c.sendErr("Failed Account Registration") return ErrBadAccount } + } else if c.acc == nil { + // By default register with the global account. + c.registerWithAccount(srv.gacc) } + } // Check client protocol request if it exists. @@ -1156,55 +1178,6 @@ func (c *client) processPong() { c.mu.Unlock() } -func (c *client) processMsgArgs(arg []byte) error { - if c.trace { - c.traceInOp("MSG", arg) - } - - // Unroll splitArgs to avoid runtime/heap issues - a := [MAX_MSG_ARGS][]byte{} - args := a[:0] - start := -1 - for i, b := range arg { - switch b { - case ' ', '\t', '\r', '\n': - if start >= 0 { - args = append(args, arg[start:i]) - start = -1 - } - default: - if start < 0 { - start = i - } - } - } - if start >= 0 { - args = append(args, arg[start:]) - } - - switch len(args) { - case 3: - c.pa.reply = nil - c.pa.szb = args[2] - c.pa.size = parseSize(args[2]) - case 4: - c.pa.reply = args[2] - c.pa.szb = args[3] - c.pa.size = parseSize(args[3]) - default: - return fmt.Errorf("processMsgArgs Parse Error: '%s'", arg) - } - if c.pa.size < 0 { - return fmt.Errorf("processMsgArgs Bad or Missing Size: '%s'", arg) - } - - // Common ones processed after check for arg length - c.pa.subject = args[0] - c.pa.sid = args[1] - - return nil -} - func (c *client) processPub(arg []byte) error { if c.trace { c.traceInOp("PUB", arg) @@ -1231,6 +1204,7 @@ func (c *client) processPub(arg []byte) error { args = append(args, arg[start:]) } + c.pa.arg = arg switch len(args) { case 2: c.pa.subject = args[0] @@ -1290,6 +1264,7 @@ func (c *client) processSub(argo []byte) (err error) { c.in.subs++ // Copy so we do not reference a potentially large buffer + // FIXME(dlc) - make more efficient. arg := make([]byte, len(argo)) copy(arg, argo) args := splitArg(arg) @@ -1307,16 +1282,17 @@ func (c *client) processSub(argo []byte) (err error) { return fmt.Errorf("processSub Parse Error: '%s'", arg) } - shouldForward := false - c.mu.Lock() if c.nc == nil { c.mu.Unlock() return nil } + // Grab connection type. + ctype := c.typ + // Check permissions if applicable. - if c.typ == ROUTER { + if ctype == ROUTER { if !c.canExport(sub.subject) { c.mu.Unlock() return nil @@ -1339,19 +1315,15 @@ func (c *client) processSub(argo []byte) (err error) { // We can have two SUB protocols coming from a route due to some // race conditions. We should make sure that we process only one. sid := string(sub.sid) - var chkImports bool + acc := c.acc + // Subscribe here. if c.subs[sid] == nil { c.subs[sid] = sub - if c.sl != nil { - err = c.sl.Insert(sub) + if acc != nil && acc.sl != nil { + err = acc.sl.Insert(sub) if err != nil { delete(c.subs, sid) - } else { - if c.acc != nil { - chkImports = true - } - shouldForward = c.typ != ROUTER } } } @@ -1363,13 +1335,15 @@ func (c *client) processSub(argo []byte) (err error) { } else if c.opts.Verbose { c.sendOK() } - if chkImports { - if err := c.addShadowSubscriptions(sub); err != nil { + + if acc != nil { + if err := c.addShadowSubscriptions(acc, sub); err != nil { c.Errorf(err.Error()) } - } - if shouldForward { - c.srv.broadcastSubscribe(sub) + // If we are routing and this is a local sub, add to the route map for the associated account. + if ctype == CLIENT { + c.srv.updateRouteSubscriptionMap(acc, sub, 1) + } } return nil @@ -1378,23 +1352,20 @@ func (c *client) processSub(argo []byte) (err error) { // If the client's account has stream imports and there are matches for // this subscription's subject, then add shadow subscriptions in // other accounts that can export this subject. -func (c *client) addShadowSubscriptions(sub *subscription) error { - c.mu.Lock() - acc := c.acc - c.mu.Unlock() - +func (c *client) addShadowSubscriptions(acc *Account, sub *subscription) error { if acc == nil { return ErrMissingAccount } - subject := string(sub.subject) - tokens := strings.Split(subject, tsep) - var rims [32]*streamImport var ims = rims[:0] + var tokens []string acc.mu.RLock() for _, im := range acc.imports.streams { + if tokens == nil { + tokens = strings.Split(string(sub.subject), tsep) + } if isSubsetMatch(tokens, im.prefix+im.from) { ims = append(ims, im) } @@ -1414,20 +1385,29 @@ func (c *client) addShadowSubscriptions(sub *subscription) error { // Just remove prefix from what they gave us. That maps into other space. nsub.subject = sub.subject[len(im.prefix):] } + + c.Debugf("Creating import subscription on %q from account %q", nsub.subject, im.acc.Name) + if err := im.acc.sl.Insert(&nsub); err != nil { errs := fmt.Sprintf("Could not add shadow import subscription for account %q", im.acc.Name) c.Debugf(errs) return fmt.Errorf(errs) } + // Update our route map here. + c.srv.updateRouteSubscriptionMap(im.acc, &nsub, 1) + // FIXME(dlc) - make sure to remove as well! + if shadow == nil { shadow = make([]*subscription, 0, len(ims)) } shadow = append(shadow, &nsub) } - c.mu.Lock() - sub.shadow = shadow - c.mu.Unlock() + if shadow != nil { + c.mu.Lock() + sub.shadow = shadow + c.mu.Unlock() + } return nil } @@ -1467,23 +1447,25 @@ func (c *client) unsubscribe(sub *subscription, force bool) { c.traceOp("<-> %s", "DELSUB", sub.sid) delete(c.subs, string(sub.sid)) - if c.sl != nil { - c.sl.Remove(sub) + var acc *Account + if c.typ == CLIENT { + acc = c.acc + } else if i := bytes.Index(sub.sid, []byte(" ")); i > 0 { + // First part of SID for route is account name. + acc = c.srv.LookupAccount(string(sub.sid[:i])) + c.removeReplySubTimeout(sub) } - // If we are a queue subscriber on a client connection and we have routes, - // we will remember the remote sid and the queue group in case a route - // tries to deliver us a message. Remote queue subscribers are directed - // so we need to know what to do to avoid unnecessary message drops - // from [auto-]unsubscribe. - if c.typ == CLIENT && c.srv != nil && len(sub.queue) > 0 { - c.srv.holdRemoteQSub(sub) + if acc != nil { + acc.sl.Remove(sub) } - // Check to see if we have shadown subscriptions. + // Check to see if we have shadow subscriptions. for _, nsub := range sub.shadow { if err := nsub.im.acc.sl.Remove(nsub); err != nil { c.Debugf("Could not remove shadow import subscription for account %q", nsub.im.acc.Name) + } else if c.typ == CLIENT && c.srv != nil { + c.srv.updateRouteSubscriptionMap(nsub.im.acc, nsub, -1) } } sub.shadow = nil @@ -1504,57 +1486,62 @@ func (c *client) processUnsub(arg []byte) error { default: return fmt.Errorf("processUnsub Parse Error: '%s'", arg) } - // Indicate activity. c.in.subs += 1 var sub *subscription unsub := false - shouldForward := false ok := false c.mu.Lock() + + // Grab connection type. + ctype := c.typ + var acc *Account + if sub, ok = c.subs[string(sid)]; ok { if max > 0 { sub.max = int64(max) } else { // Clear it here to override sub.max = 0 + unsub = true } - unsub = true - shouldForward = c.typ != ROUTER && c.srv != nil + acc = c.acc } c.mu.Unlock() - if unsub { - c.unsubscribe(sub, false) - } - if shouldForward { - c.srv.broadcastUnSubscribe(sub) - } if c.opts.Verbose { c.sendOK() } + if unsub { + c.unsubscribe(sub, false) + if acc != nil && ctype == CLIENT { + c.srv.updateRouteSubscriptionMap(acc, sub, -1) + } + } + return nil } func (c *client) msgHeader(mh []byte, sub *subscription, reply []byte) []byte { - mh = append(mh, sub.sid...) - mh = append(mh, ' ') + if len(sub.sid) > 0 { + mh = append(mh, sub.sid...) + mh = append(mh, ' ') + } if reply != nil { mh = append(mh, reply...) mh = append(mh, ' ') } mh = append(mh, c.pa.szb...) - mh = append(mh, "\r\n"...) + mh = append(mh, _CRLF_...) return mh } // Used to treat maps as efficient set var needFlush = struct{}{} -var routeSeen = struct{}{} func (c *client) deliverMsg(sub *subscription, mh, msg []byte) bool { if sub.client == nil { @@ -1584,7 +1571,7 @@ func (c *client) deliverMsg(sub *subscription, mh, msg []byte) bool { // Due to defer, reverse the code order so that execution // is consistent with other cases where we unsubscribe. if shouldForward { - defer srv.broadcastUnSubscribe(sub) + defer srv.updateRouteSubscriptionMap(client.acc, sub, -1) } defer client.unsubscribe(sub, true) } else if sub.nm > sub.max { @@ -1592,7 +1579,7 @@ func (c *client) deliverMsg(sub *subscription, mh, msg []byte) bool { client.mu.Unlock() client.unsubscribe(sub, true) if shouldForward { - srv.broadcastUnSubscribe(sub) + srv.updateRouteSubscriptionMap(client.acc, sub, -1) } return false } @@ -1658,20 +1645,14 @@ func (c *client) prunePubPermsCache() { // pubAllowed checks on publish permissioning. func (c *client) pubAllowed(subject []byte) bool { - // Disallow publish to _SYS.>, these are reserved for internals. - if len(subject) > 4 && string(subject[:5]) == "_SYS." { - return false - } if c.perms == nil { return true } - // Check if published subject is allowed if we have permissions in place. allowed, ok := c.perms.pcache[string(subject)] if ok { return allowed } - // Cache miss, check allow then deny as needed. if c.perms.pub.allow != nil { r := c.perms.pub.allow.Match(string(subject)) @@ -1685,10 +1666,8 @@ func (c *client) pubAllowed(subject []byte) bool { r := c.perms.pub.deny.Match(string(subject)) allowed = len(r.psubs) == 0 } - // Update our cache here. c.perms.pcache[string(subject)] = allowed - // Prune if needed. if len(c.perms.pcache) > maxPermCacheSize { c.prunePubPermsCache() @@ -1698,14 +1677,14 @@ func (c *client) pubAllowed(subject []byte) bool { // Used to mimic client like replies. const ( - replyPrefix = "_INBOX." + replyPrefix = "_R_." replyPrefixLen = len(replyPrefix) digits = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" base = 62 ) // newServiceReply is used when rewriting replies that cross account boundaries. -// These will look like _INBOX.XXXXXXXX, similar to the old style of replies for most clients. +// These will look like _R_.XXXXXXXX. func (c *client) newServiceReply() []byte { // Check to see if we have our own rand yet. Global rand // has contention with lots of clients, etc. @@ -1713,7 +1692,7 @@ func (c *client) newServiceReply() []byte { c.in.prand = rand.New(rand.NewSource(time.Now().UnixNano())) } - var b = [15]byte{'_', 'I', 'N', 'B', 'O', 'X', '.'} + var b = [15]byte{'_', 'R', '_', '.'} rn := c.in.prand.Int63() for i, l := replyPrefixLen, rn; i < len(b); i++ { b[i] = digits[l%base] @@ -1722,11 +1701,22 @@ func (c *client) newServiceReply() []byte { return b[:] } -// processMsg is called to process an inbound msg from a client. -func (c *client) processInboundMsg(msg []byte) { - // Snapshot server. - srv := c.srv +// Test whether a reply subject is a service import reply. +func isServiceReply(reply []byte) bool { + return len(reply) > 3 && string(reply[:4]) == replyPrefix +} +// This will decide to call the client code or router code. +func (c *client) processInboundMsg(msg []byte) { + if c.typ == CLIENT { + c.processInboundClientMsg(msg) + } else { + c.processInboundRouteMsg(msg) + } +} + +// processInboundClientMsg is called to process an inbound msg from a client. +func (c *client) processInboundClientMsg(msg []byte) { // Update statistics // The msg includes the CR_LF, so pull back out for accounting. c.in.msgs += 1 @@ -1736,18 +1726,24 @@ func (c *client) processInboundMsg(msg []byte) { c.traceMsg(msg) } - // Check pub permissions (don't do this for routes) - if c.typ == CLIENT && !c.pubAllowed(c.pa.subject) { + // Check pub permissions + if c.perms != nil && !c.pubAllowed(c.pa.subject) { c.pubPermissionViolation(c.pa.subject) return } + // Now check for reserved replies. These are used for service imports. + if isServiceReply(c.pa.reply) { + c.replySubjectViolation(c.pa.reply) + return + } + if c.opts.Verbose { c.sendOK() } // Mostly under testing scenarios. - if srv == nil { + if c.srv == nil || c.acc == nil { return } @@ -1756,8 +1752,7 @@ func (c *client) processInboundMsg(msg []byte) { var r *SublistResult var ok bool - genid := atomic.LoadUint64(&c.sl.genid) - + genid := atomic.LoadUint64(&c.acc.sl.genid) if genid == c.in.genid && c.in.results != nil { r, ok = c.in.results[string(c.pa.subject)] } else { @@ -1766,10 +1761,11 @@ func (c *client) processInboundMsg(msg []byte) { c.in.genid = genid } + // Go back to the sublist data structure. if !ok { - r = c.sl.Match(string(c.pa.subject)) + r = c.acc.sl.Match(string(c.pa.subject)) c.in.results[string(c.pa.subject)] = r - // Prune the results cache. Keeps us from unbounded growth. + // Prune the results cache. Keeps us from unbounded growth. Random delete. if len(c.in.results) > maxResultCacheSize { n := 0 for subject := range c.in.results { @@ -1781,85 +1777,77 @@ func (c *client) processInboundMsg(msg []byte) { } } - // Check to see if we need to route this message to - // another account. - if c.typ == CLIENT && c.acc != nil && c.acc.imports.services != nil { - c.acc.mu.RLock() - rm := c.acc.imports.services[string(c.pa.subject)] - c.acc.mu.RUnlock() - // Get the results from the other account for the mapped "to" subject. - if rm != nil && rm.acc != nil && rm.acc.sl != nil { - var nrr []byte - if rm.ae { - c.acc.removeServiceImport(rm.from) - } - if c.pa.reply != nil { - // We want to remap this to provide anonymity. - nrr = c.newServiceReply() - rm.acc.addImplicitServiceImport(c.acc, string(nrr), string(c.pa.reply), true) - } - // FIXME(dlc) - Do L1 cache trick from above. - rr := rm.acc.sl.Match(rm.to) - c.processMsgResults(rr, msg, []byte(rm.to), nrr) - } + // Check to see if we need to map/route to another account. + if c.acc.imports.services != nil { + c.checkForImportServices(c.acc, msg) } - // This is the fanout scale. - fanout := len(r.psubs) + len(r.qsubs) - // Check for no interest, short circuit if so. - if fanout == 0 { - return - } - - if c.typ == ROUTER { - c.processRoutedMsgResults(r, msg) - } else if c.typ == CLIENT { - c.processMsgResults(r, msg, c.pa.subject, c.pa.reply) + // This is the fanout scale. + if len(r.psubs)+len(r.qsubs) > 0 { + c.processMsgResults(c.acc, r, msg, c.pa.subject, c.pa.reply) } } +// This checks and process import services by doing the mapping and sending the +// message onward if applicable. +func (c *client) checkForImportServices(acc *Account, msg []byte) { + if acc == nil || acc.imports.services == nil { + return + } + acc.mu.RLock() + rm := acc.imports.services[string(c.pa.subject)] + acc.mu.RUnlock() + // Get the results from the other account for the mapped "to" subject. + if rm != nil && rm.acc != nil && rm.acc.sl != nil { + var nrr []byte + if rm.ae { + acc.removeServiceImport(rm.from) + } + if c.pa.reply != nil { + // We want to remap this to provide anonymity. + nrr = c.newServiceReply() + rm.acc.addImplicitServiceImport(acc, string(nrr), string(c.pa.reply), true) + } + // FIXME(dlc) - Do L1 cache trick from above. + rr := rm.acc.sl.Match(rm.to) + c.processMsgResults(rm.acc, rr, msg, []byte(rm.to), nrr) + } +} + +type routeTarget struct { + sub *subscription + qnames [][]byte +} + // This processes the sublist results for a given message. -func (c *client) processMsgResults(r *SublistResult, msg, subject, reply []byte) { - // msg header - msgh := c.msgb[:msgHeadProtoLen] +func (c *client) processMsgResults(acc *Account, r *SublistResult, msg, subject, reply []byte) { + // msg header for clients. + msgh := c.msgb[1:msgHeadProtoLen] msgh = append(msgh, subject...) msgh = append(msgh, ' ') si := len(msgh) - // Used to only send messages once across any given route. - var rmap map[string]struct{} + // For sending messages across routes + var rmap map[*client]*routeTarget // Loop over all normal subscriptions that match. for _, sub := range r.psubs { - // Check if this is a send to a ROUTER, make sure we only send it - // once. The other side will handle the appropriate re-processing - // and fan-out. Also enforce 1-Hop semantics, so no routing to another. - if sub.client.typ == ROUTER { - // Check to see if we have already sent it here. + // Check if this is a send to a ROUTER. We now process + // these after everything else. + if sub.client != nil && sub.client.typ == ROUTER { if rmap == nil { - rmap = make(map[string]struct{}, c.srv.numRoutes()) + rmap = map[*client]*routeTarget{} } - sub.client.mu.Lock() - if sub.client.nc == nil || - sub.client.route == nil || - sub.client.route.remoteID == "" { - c.Debugf("Bad or Missing ROUTER Identity, not processing msg") - sub.client.mu.Unlock() - continue + if c.typ != ROUTER && rmap[sub.client] == nil { + rmap[sub.client] = &routeTarget{sub: sub} } - if _, ok := rmap[sub.client.route.remoteID]; ok { - c.Debugf("Ignoring route, already processed and sent msg") - sub.client.mu.Unlock() - continue - } - rmap[sub.client.route.remoteID] = routeSeen - sub.client.mu.Unlock() + continue } - // Check for import mapped subs + // Check for stream import mapped subs if sub.im != nil && sub.im.prefix != "" { // Redo the subject here on the fly. - msgh := c.msgb[:msgHeadProtoLen] + msgh = c.msgb[1:msgHeadProtoLen] msgh = append(msgh, sub.im.prefix...) msgh = append(msgh, c.pa.subject...) msgh = append(msgh, ' ') @@ -1870,6 +1858,16 @@ func (c *client) processMsgResults(r *SublistResult, msg, subject, reply []byte) c.deliverMsg(sub, mh, msg) } + // If we are sourced from a route we need to have direct filtered queues. + if c.typ == ROUTER && c.pa.queues == nil { + return + } + + // Set these up to optionally filter based on the queue lists. + // This is for messages received from routes which will have directed + // guidance on which queue groups we should deliver to. + qf := c.pa.queues + // Check to see if we have our own rand yet. Global rand // has contention with lots of clients, etc. if c.in.prand == nil { @@ -1877,30 +1875,127 @@ func (c *client) processMsgResults(r *SublistResult, msg, subject, reply []byte) } // Process queue subs + var bounce bool + for i := 0; i < len(r.qsubs); i++ { qsubs := r.qsubs[i] + // If we have a filter check that here. We could make this a map or someting more + // complex but linear search since we expect queues to be small should be faster + // and more cache friendly. + if qf != nil && len(qsubs) > 0 { + tqn := qsubs[0].queue + for _, qn := range qf { + if bytes.Equal(qn, tqn) { + goto selectQSub + } + } + continue + } + + selectQSub: + + var rsub *subscription + // Find a subscription that is able to deliver this message // starting at a random index. startIndex := c.in.prand.Intn(len(qsubs)) for i := 0; i < len(qsubs); i++ { index := (startIndex + i) % len(qsubs) sub := qsubs[index] - if sub != nil { - // Check for mapped subs - if sub.im != nil && sub.im.prefix != "" { - // Redo the subject here on the fly. - msgh := c.msgb[:msgHeadProtoLen] - msgh = append(msgh, sub.im.prefix...) - msgh = append(msgh, c.pa.subject...) - msgh = append(msgh, ' ') - si = len(msgh) - } - mh := c.msgHeader(msgh[:si], sub, reply) - if c.deliverMsg(sub, mh, msg) { - break + if sub == nil { + continue + } + // Sending to a remote route. + if sub.client.typ == ROUTER { + if c.typ == ROUTER { + // We just came from a route, so skip and prefer local subs. + // Keep our first rsub in case all else fails. + if rsub == nil { + rsub = sub + } + continue + } else { + if rmap == nil { + rmap = map[*client]*routeTarget{} + rmap[sub.client] = &routeTarget{sub: sub, qnames: [][]byte{sub.queue}} + } else if rt := rmap[sub.client]; rt != nil { + rt.qnames = append(rt.qnames, sub.queue) + } else { + rmap[sub.client] = &routeTarget{sub: sub, qnames: [][]byte{sub.queue}} + } } + break + } + + // Check for mapped subs + if sub.im != nil && sub.im.prefix != "" { + // Redo the subject here on the fly. + msgh = c.msgb[1:msgHeadProtoLen] + msgh = append(msgh, sub.im.prefix...) + msgh = append(msgh, c.pa.subject...) + msgh = append(msgh, ' ') + si = len(msgh) + } + + mh := c.msgHeader(msgh[:si], sub, reply) + if c.deliverMsg(sub, mh, msg) { + // Clear rsub + rsub = nil + break } } + if rsub != nil { + // If we are here we tried to deliver to a local qsub + // but failed. So we will send it to a remote. + bounce = true + if rmap == nil { + rmap = map[*client]*routeTarget{} + } + if rt := rmap[rsub.client]; rt != nil { + rt.qnames = append(rt.qnames, rsub.queue) + } else { + rmap[rsub.client] = &routeTarget{sub: rsub, qnames: [][]byte{rsub.queue}} + } + } + } + + // Don't send messages to routes if we ourselves are a route. + if (c.typ != CLIENT && !bounce) || len(rmap) == 0 { + return + } + + // Now process route connections. + for _, rt := range rmap { + mh := c.msgb[:msgHeadProtoLen] + mh = append(mh, acc.Name...) + mh = append(mh, ' ') + mh = append(mh, subject...) + mh = append(mh, ' ') + // If we have queues the third token turns into marker + // that signals number of queues. The leading byte signifies + // whether a reply is present as well. + if len(rt.qnames) > 0 { + if reply != nil { + mh = append(mh, '+') // Signal that there is a reply. + } else { + mh = append(mh, '|') // Only queues + } + mh = append(mh, ' ') + if reply != nil { + mh = append(mh, reply...) + mh = append(mh, ' ') + } + for _, qn := range rt.qnames { + mh = append(mh, qn...) + mh = append(mh, ' ') + } + } else if reply != nil { + mh = append(mh, reply...) + mh = append(mh, ' ') + } + mh = append(mh, c.pa.szb...) + mh = append(mh, _CRLF_...) + c.deliverMsg(rt.sub, mh, msg) } } @@ -1909,6 +2004,11 @@ func (c *client) pubPermissionViolation(subject []byte) { c.Errorf("Publish Violation - User %q, Subject %q", c.opts.Username, subject) } +func (c *client) replySubjectViolation(reply []byte) { + c.sendErr(fmt.Sprintf("Permissions Violation for Publish with Reply of %q", reply)) + c.Errorf("Publish Violation - User %q, Reply %q", c.opts.Username, reply) +} + func (c *client) processPingTimer() { c.mu.Lock() defer c.mu.Unlock() @@ -1960,7 +2060,7 @@ func (c *client) clearPingTimer() { // Lock should be held func (c *client) setAuthTimer(d time.Duration) { - c.atmr = time.AfterFunc(d, func() { c.authTimeout() }) + c.atmr = time.AfterFunc(d, c.authTimeout) } // Lock should be held @@ -2031,7 +2131,7 @@ func (c *client) processSubsOnConfigReload(awcsti map[string]struct{}) { checkPerms = c.perms != nil checkAcc = c.acc != nil ) - if c.sl == nil || (!checkPerms && !checkAcc) { + if !checkPerms && !checkAcc { c.mu.Unlock() return } @@ -2073,7 +2173,7 @@ func (c *client) processSubsOnConfigReload(awcsti map[string]struct{}) { oldShadows := sub.shadow sub.shadow = nil c.mu.Unlock() - c.addShadowSubscriptions(sub) + c.addShadowSubscriptions(c.acc, sub) for _, nsub := range oldShadows { nsub.im.acc.sl.Remove(nsub) } @@ -2089,6 +2189,12 @@ func (c *client) processSubsOnConfigReload(awcsti map[string]struct{}) { } } +// Allows us to count up all the queue subscribers during close. +type qsub struct { + sub *subscription + n int32 +} + func (c *client) closeConnection(reason ClosedState) { c.mu.Lock() if c.nc == nil { @@ -2103,12 +2209,19 @@ func (c *client) closeConnection(reason ClosedState) { c.clearConnection(reason) c.nc = nil - // Snapshot for use. - subs := make([]*subscription, 0, len(c.subs)) - for _, sub := range c.subs { - // Auto-unsubscribe subscriptions must be unsubscribed forcibly. - sub.max = 0 - subs = append(subs, sub) + ctype := c.typ + + // Snapshot for use if we are a client connection. + // FIXME(dlc) - we can just stub in a new one for client + // and reference existing one. + var subs []*subscription + if ctype == CLIENT { + subs = make([]*subscription, 0, len(c.subs)) + for _, sub := range c.subs { + // Auto-unsubscribe subscriptions must be unsubscribed forcibly. + sub.max = 0 + subs = append(subs, sub) + } } srv := c.srv @@ -2125,10 +2238,15 @@ func (c *client) closeConnection(reason ClosedState) { connectURLs = c.route.connectURLs } + acc := c.acc c.mu.Unlock() // Remove clients subscriptions. - c.sl.RemoveBatch(subs) + if ctype == CLIENT { + acc.sl.RemoveBatch(subs) + } else { + go c.removeRemoteSubs() + } if srv != nil { // This is a route that disconnected, but we are not in lame duck mode... @@ -2143,10 +2261,33 @@ func (c *client) closeConnection(reason ClosedState) { // Unregister srv.removeClient(c) - // Remove remote subscriptions. - if c.typ != ROUTER { - // Forward UNSUBs protocols to all routes - srv.broadcastUnSubscribeBatch(subs) + // Update remote subscriptions. + if acc != nil && ctype == CLIENT { + qsubs := map[string]*qsub{} + for _, sub := range subs { + if sub.queue == nil { + srv.updateRouteSubscriptionMap(acc, sub, -1) + } else { + // We handle queue subscribers special in case we + // have a bunch we can just send one update to the + // connected routes. + key := string(sub.subject) + " " + string(sub.queue) + if esub, ok := qsubs[key]; ok { + esub.n += 1 + } else { + qsubs[key] = &qsub{sub, 1} + } + } + } + // Process any qsubs here. + for _, esub := range qsubs { + srv.updateRouteSubscriptionMap(acc, esub.sub, -(esub.n)) + } + if prev := c.acc.removeClient(c); prev == 1 && c.srv != nil { + c.srv.mu.Lock() + c.srv.activeAccounts-- + c.srv.mu.Unlock() + } } } diff --git a/server/client_test.go b/server/client_test.go index afa2efeb..fa0b5d18 100644 --- a/server/client_test.go +++ b/server/client_test.go @@ -663,12 +663,12 @@ func TestClientRemoveSubsOnDisconnect(t *testing.T) { }() <-ch - if c.sl.Count() != 2 { - t.Fatalf("Should have 2 subscriptions, got %d\n", c.sl.Count()) + if s.NumSubscriptions() != 2 { + t.Fatalf("Should have 2 subscriptions, got %d\n", s.NumSubscriptions()) } c.closeConnection(ClientClosed) - if c.sl.Count() != 0 { - t.Fatalf("Should have no subscriptions after close, got %d\n", s.gsl.Count()) + if s.NumSubscriptions() != 0 { + t.Fatalf("Should have no subscriptions after close, got %d\n", s.NumSubscriptions()) } } @@ -684,8 +684,8 @@ func TestClientDoesNotAddSubscriptionsWhenConnectionClosed(t *testing.T) { }() <-ch - if c.sl.Count() != 0 { - t.Fatalf("Should have no subscriptions after close, got %d\n", c.sl.Count()) + if c.acc.sl.Count() != 0 { + t.Fatalf("Should have no subscriptions after close, got %d\n", c.acc.sl.Count()) } } diff --git a/server/const.go b/server/const.go index f807fc48..dedb0962 100644 --- a/server/const.go +++ b/server/const.go @@ -38,7 +38,7 @@ var ( const ( // VERSION is the current version for the server. - VERSION = "1.3.1" + VERSION = "2.0-alpha" // PROTO is the currently supported protocol. // 0 was the original diff --git a/server/log.go b/server/log.go index 90ee1a14..60e29ff2 100644 --- a/server/log.go +++ b/server/log.go @@ -52,6 +52,10 @@ func (s *Server) ConfigureLogger() { opts = s.getOpts() ) + if opts.NoLog { + return + } + syslog := opts.Syslog if isWindowsService() && opts.LogFile == "" { // Enable syslog if no log file is specified and we're running as a diff --git a/server/monitor.go b/server/monitor.go index f8222f2e..e864222d 100644 --- a/server/monitor.go +++ b/server/monitor.go @@ -711,14 +711,14 @@ func (s *Server) Subsz(opts *SubszOptions) (*Subsz, error) { } // FIXME(dlc) - Make account aware. - sz := &Subsz{s.gsl.Stats(), 0, offset, limit, nil} + sz := &Subsz{s.gacc.sl.Stats(), 0, offset, limit, nil} if subdetail { // Now add in subscription's details var raw [4096]*subscription subs := raw[:0] - s.gsl.localSubs(&subs) + s.gacc.sl.localSubs(&subs) details := make([]SubDetail, len(subs)) i := 0 // TODO(dlc) - may be inefficient and could just do normal match when total subs is large and filtering. @@ -940,7 +940,7 @@ func (s *Server) Varz(varzOpts *VarzOptions) (*Varz, error) { v.SlowConsumers = atomic.LoadInt64(&s.slowConsumers) v.MaxPending = opts.MaxPending v.WriteDeadline = opts.WriteDeadline - v.Subscriptions = s.gsl.Count() + v.Subscriptions = s.gacc.sl.Count() v.ConfigLoadTime = s.configTime // Need a copy here since s.httpReqStats can change while doing // the marshaling down below. diff --git a/server/opts.go b/server/opts.go index 4ce93479..308210da 100644 --- a/server/opts.go +++ b/server/opts.go @@ -93,7 +93,7 @@ type Options struct { TLSCaCert string `json:"-"` TLSConfig *tls.Config `json:"-"` WriteDeadline time.Duration `json:"-"` - RQSubsSweep time.Duration `json:"-"` + RQSubsSweep time.Duration `json:"-"` // Deprecated MaxClosedClients int `json:"-"` LameDuckDuration time.Duration `json:"-"` @@ -772,7 +772,7 @@ func parseAccounts(v interface{}, opts *Options, errors *[]error, warnings *[]er } accounts = append(accounts, ta) } - if err := stream.acc.addStreamExport(stream.sub, accounts); err != nil { + if err := stream.acc.AddStreamExport(stream.sub, accounts); err != nil { msg := fmt.Sprintf("Error adding stream export %q: %v", stream.sub, err) *errors = append(*errors, &configErr{tk, msg}) continue @@ -790,7 +790,7 @@ func parseAccounts(v interface{}, opts *Options, errors *[]error, warnings *[]er } accounts = append(accounts, ta) } - if err := service.acc.addServiceExport(service.sub, accounts); err != nil { + if err := service.acc.AddServiceExport(service.sub, accounts); err != nil { msg := fmt.Sprintf("Error adding service export %q: %v", service.sub, err) *errors = append(*errors, &configErr{tk, msg}) continue @@ -803,7 +803,7 @@ func parseAccounts(v interface{}, opts *Options, errors *[]error, warnings *[]er *errors = append(*errors, &configErr{tk, msg}) continue } - if err := stream.acc.addStreamImport(ta, stream.sub, stream.pre); err != nil { + if err := stream.acc.AddStreamImport(ta, stream.sub, stream.pre); err != nil { msg := fmt.Sprintf("Error adding stream import %q: %v", stream.sub, err) *errors = append(*errors, &configErr{tk, msg}) continue @@ -819,7 +819,7 @@ func parseAccounts(v interface{}, opts *Options, errors *[]error, warnings *[]er if service.to == "" { service.to = service.sub } - if err := service.acc.addServiceImport(ta, service.to, service.sub); err != nil { + if err := service.acc.AddServiceImport(ta, service.to, service.sub); err != nil { msg := fmt.Sprintf("Error adding service import %q: %v", service.sub, err) *errors = append(*errors, &configErr{tk, msg}) continue diff --git a/server/parser.go b/server/parser.go index 6eec1acc..704b57e0 100644 --- a/server/parser.go +++ b/server/parser.go @@ -18,11 +18,15 @@ import ( ) type pubArg struct { + arg []byte + account []byte + queues [][]byte + //cacheKey []byte subject []byte reply []byte - sid []byte - szb []byte - size int + // sid []byte + szb []byte + size int } type parseState struct { @@ -73,6 +77,15 @@ const ( OP_SUB OP_SUB_SPC SUB_ARG + OP_A + OP_ASUB + OP_ASUB_SPC + ASUB_ARG + OP_AUSUB + OP_AUSUB_SPC + AUSUB_ARG + OP_R + OP_RS OP_U OP_UN OP_UNS @@ -121,11 +134,17 @@ func (c *client) parse(buf []byte) error { c.state = OP_S case 'U', 'u': c.state = OP_U - case 'M', 'm': + case 'R', 'r': if c.typ == CLIENT { goto parseErr } else { - c.state = OP_M + c.state = OP_R + } + case 'A', 'a': + if c.typ == CLIENT { + goto parseErr + } else { + c.state = OP_A } case 'C', 'c': c.state = OP_C @@ -243,6 +262,85 @@ func (c *client) parse(buf []byte) error { } continue } + case OP_A: + switch b { + case '+': + c.state = OP_ASUB + case '-', 'u': + c.state = OP_AUSUB + default: + goto parseErr + } + case OP_ASUB: + switch b { + case ' ', '\t': + c.state = OP_ASUB_SPC + default: + goto parseErr + } + case OP_ASUB_SPC: + switch b { + case ' ', '\t': + continue + default: + c.state = ASUB_ARG + c.as = i + } + case ASUB_ARG: + switch b { + case '\r': + c.drop = 1 + case '\n': + var arg []byte + if c.argBuf != nil { + arg = c.argBuf + c.argBuf = nil + } else { + arg = buf[c.as : i-c.drop] + } + if err := c.processAccountSub(arg); err != nil { + return err + } + c.drop, c.as, c.state = 0, i+1, OP_START + default: + if c.argBuf != nil { + c.argBuf = append(c.argBuf, b) + } + } + case OP_AUSUB: + switch b { + case ' ', '\t': + c.state = OP_AUSUB_SPC + default: + goto parseErr + } + case OP_AUSUB_SPC: + switch b { + case ' ', '\t': + continue + default: + c.state = AUSUB_ARG + c.as = i + } + case AUSUB_ARG: + switch b { + case '\r': + c.drop = 1 + case '\n': + var arg []byte + if c.argBuf != nil { + arg = c.argBuf + c.argBuf = nil + } else { + arg = buf[c.as : i-c.drop] + } + c.processAccountUnsub(arg) + c.drop, c.as, c.state = 0, i+1, OP_START + default: + if c.argBuf != nil { + c.argBuf = append(c.argBuf, b) + } + } case OP_S: switch b { case 'U', 'u': @@ -284,7 +382,13 @@ func (c *client) parse(buf []byte) error { } else { arg = buf[c.as : i-c.drop] } - if err := c.processSub(arg); err != nil { + var err error + if c.typ == CLIENT { + err = c.processSub(arg) + } else { + err = c.processRemoteSub(arg) + } + if err != nil { return err } c.drop, c.as, c.state = 0, i+1, OP_START @@ -293,6 +397,24 @@ func (c *client) parse(buf []byte) error { c.argBuf = append(c.argBuf, b) } } + case OP_R: + switch b { + case 'S', 's': + c.state = OP_RS + case 'M', 'm': + c.state = OP_M + default: + goto parseErr + } + case OP_RS: + switch b { + case '+': + c.state = OP_SUB + case '-': + c.state = OP_UNSUB + default: + goto parseErr + } case OP_U: switch b { case 'N', 'n': @@ -348,7 +470,13 @@ func (c *client) parse(buf []byte) error { } else { arg = buf[c.as : i-c.drop] } - if err := c.processUnsub(arg); err != nil { + var err error + if c.typ == CLIENT { + err = c.processUnsub(arg) + } else { + err = c.processRemoteUnsub(arg) + } + if err != nil { return err } c.drop, c.as, c.state = 0, i+1, OP_START @@ -510,7 +638,7 @@ func (c *client) parse(buf []byte) error { } else { arg = buf[c.as : i-c.drop] } - if err := c.processMsgArgs(arg); err != nil { + if err := c.processRoutedMsgArgs(arg); err != nil { return err } c.drop, c.as, c.state = 0, i+1, MSG_PAYLOAD @@ -655,6 +783,7 @@ func (c *client) parse(buf []byte) error { // Check for split buffer scenarios for any ARG state. if c.state == SUB_ARG || c.state == UNSUB_ARG || c.state == PUB_ARG || + c.state == ASUB_ARG || c.state == AUSUB_ARG || c.state == MSG_ARG || c.state == MINUS_ERR_ARG || c.state == CONNECT_ARG || c.state == INFO_ARG { // Setup a holder buffer to deal with split buffer scenario. @@ -729,21 +858,14 @@ func protoSnippet(start int, buf []byte) string { // clonePubArg is used when the split buffer scenario has the pubArg in the existing read buffer, but // we need to hold onto it into the next read. func (c *client) clonePubArg() { + // Just copy and re-process original arg buffer. c.argBuf = c.scratch[:0] - c.argBuf = append(c.argBuf, c.pa.subject...) - c.argBuf = append(c.argBuf, c.pa.reply...) - c.argBuf = append(c.argBuf, c.pa.sid...) - c.argBuf = append(c.argBuf, c.pa.szb...) + c.argBuf = append(c.argBuf, c.pa.arg...) - c.pa.subject = c.argBuf[:len(c.pa.subject)] - - if c.pa.reply != nil { - c.pa.reply = c.argBuf[len(c.pa.subject) : len(c.pa.subject)+len(c.pa.reply)] + // This is a routed msg + if c.pa.account != nil { + c.processRoutedMsgArgs(c.argBuf) + } else { + c.processPub(c.argBuf) } - - if c.pa.sid != nil { - c.pa.sid = c.argBuf[len(c.pa.subject)+len(c.pa.reply) : len(c.pa.subject)+len(c.pa.reply)+len(c.pa.sid)] - } - - c.pa.szb = c.argBuf[len(c.pa.subject)+len(c.pa.reply)+len(c.pa.sid):] } diff --git a/server/parser_test.go b/server/parser_test.go index 95631b08..fa8eadba 100644 --- a/server/parser_test.go +++ b/server/parser_test.go @@ -316,14 +316,22 @@ func TestParsePubBadSize(t *testing.T) { } } -func TestParseMsg(t *testing.T) { +func TestParseRouteMsg(t *testing.T) { c := dummyRouteClient() - pub := []byte("MSG foo RSID:1:2 5\r\nhello\r") + pub := []byte("MSG $foo foo 5\r\nhello\r") err := c.parse(pub) + if err == nil { + t.Fatalf("Expected an error, got none") + } + pub = []byte("RMSG $foo foo 5\r\nhello\r") + err = c.parse(pub) if err != nil || c.state != MSG_END { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } + if !bytes.Equal(c.pa.account, []byte("$foo")) { + t.Fatalf("Did not parse account correctly: '$foo' vs '%s'\n", c.pa.account) + } if !bytes.Equal(c.pa.subject, []byte("foo")) { t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) } @@ -333,18 +341,18 @@ func TestParseMsg(t *testing.T) { if c.pa.size != 5 { t.Fatalf("Did not parse msg size correctly: 5 vs %d\n", c.pa.size) } - if !bytes.Equal(c.pa.sid, []byte("RSID:1:2")) { - t.Fatalf("Did not parse sid correctly: 'RSID:1:2' vs '%s'\n", c.pa.sid) - } // Clear snapshots c.argBuf, c.msgBuf, c.state = nil, nil, OP_START - pub = []byte("MSG foo.bar RSID:1:2 INBOX.22 11\r\nhello world\r") + pub = []byte("RMSG $G foo.bar INBOX.22 11\r\nhello world\r") err = c.parse(pub) if err != nil || c.state != MSG_END { t.Fatalf("Unexpected: %d : %v\n", c.state, err) } + if !bytes.Equal(c.pa.account, []byte("$G")) { + t.Fatalf("Did not parse account correctly: '$G' vs '%s'\n", c.pa.account) + } if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) } @@ -354,44 +362,54 @@ func TestParseMsg(t *testing.T) { if c.pa.size != 11 { t.Fatalf("Did not parse msg size correctly: 11 vs %d\n", c.pa.size) } -} -func testMsgArg(c *client, t *testing.T) { - if !bytes.Equal(c.pa.subject, []byte("foobar")) { - t.Fatalf("Mismatched subject: '%s'\n", c.pa.subject) - } - if !bytes.Equal(c.pa.szb, []byte("22")) { - t.Fatalf("Bad size buf: '%s'\n", c.pa.szb) - } - if c.pa.size != 22 { - t.Fatalf("Bad size: %d\n", c.pa.size) - } - if !bytes.Equal(c.pa.sid, []byte("RSID:22:1")) { - t.Fatalf("Bad sid: '%s'\n", c.pa.sid) - } -} + // Clear snapshots + c.argBuf, c.msgBuf, c.state = nil, nil, OP_START -func TestParseMsgArg(t *testing.T) { - c := dummyClient() - if err := c.processMsgArgs([]byte("foobar RSID:22:1 22")); err != nil { - t.Fatalf("Unexpected parse error: %v\n", err) + pub = []byte("RMSG $G foo.bar + reply baz 11\r\nhello world\r") + err = c.parse(pub) + if err != nil || c.state != MSG_END { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) } - testMsgArg(c, t) - if err := c.processMsgArgs([]byte(" foobar RSID:22:1 22")); err != nil { - t.Fatalf("Unexpected parse error: %v\n", err) + if !bytes.Equal(c.pa.account, []byte("$G")) { + t.Fatalf("Did not parse account correctly: '$G' vs '%s'\n", c.pa.account) } - testMsgArg(c, t) - if err := c.processMsgArgs([]byte(" foobar RSID:22:1 22 ")); err != nil { - t.Fatalf("Unexpected parse error: %v\n", err) + if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { + t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) } - testMsgArg(c, t) - if err := c.processMsgArgs([]byte("foobar RSID:22:1 \t22")); err != nil { - t.Fatalf("Unexpected parse error: %v\n", err) + if !bytes.Equal(c.pa.reply, []byte("reply")) { + t.Fatalf("Did not parse reply correctly: 'reply' vs '%s'\n", c.pa.reply) } - if err := c.processMsgArgs([]byte("foobar\t\tRSID:22:1\t22\r")); err != nil { - t.Fatalf("Unexpected parse error: %v\n", err) + if len(c.pa.queues) != 1 { + t.Fatalf("Expected 1 queue, got %d", len(c.pa.queues)) + } + if !bytes.Equal(c.pa.queues[0], []byte("baz")) { + t.Fatalf("Did not parse queues correctly: 'baz' vs '%q'\n", c.pa.queues[0]) + } + + // Clear snapshots + c.argBuf, c.msgBuf, c.state = nil, nil, OP_START + + pub = []byte("RMSG $G foo.bar | baz 11\r\nhello world\r") + err = c.parse(pub) + if err != nil || c.state != MSG_END { + t.Fatalf("Unexpected: %d : %v\n", c.state, err) + } + if !bytes.Equal(c.pa.account, []byte("$G")) { + t.Fatalf("Did not parse account correctly: '$G' vs '%s'\n", c.pa.account) + } + if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { + t.Fatalf("Did not parse subject correctly: 'foo' vs '%s'\n", c.pa.subject) + } + if !bytes.Equal(c.pa.reply, []byte("")) { + t.Fatalf("Did not parse reply correctly: '' vs '%s'\n", c.pa.reply) + } + if len(c.pa.queues) != 1 { + t.Fatalf("Expected 1 queue, got %d", len(c.pa.queues)) + } + if !bytes.Equal(c.pa.queues[0], []byte("baz")) { + t.Fatalf("Did not parse queues correctly: 'baz' vs '%q'\n", c.pa.queues[0]) } - testMsgArg(c, t) } func TestParseMsgSpace(t *testing.T) { diff --git a/server/reload.go b/server/reload.go index e85ad4d8..5ea5332b 100644 --- a/server/reload.go +++ b/server/reload.go @@ -818,7 +818,7 @@ func (s *Server) reloadClusterPermissions() { for i, route := range s.routes { // Count the number of routes that can understand receiving INFO updates. route.mu.Lock() - if route.opts.Protocol >= routeProtoInfo { + if route.opts.Protocol >= RouteProtoInfo { withNewProto++ } route.mu.Unlock() @@ -865,7 +865,7 @@ func (s *Server) reloadClusterPermissions() { deleteRoutedSubs []*subscription ) // FIXME(dlc) - Change for accounts. - s.gsl.localSubs(&localSubs) + s.gacc.sl.localSubs(&localSubs) // Go through all local subscriptions for _, sub := range localSubs { @@ -887,7 +887,7 @@ func (s *Server) reloadClusterPermissions() { for _, route := range routes { route.mu.Lock() // If route is to older server, simply close connection. - if route.opts.Protocol < routeProtoInfo { + if route.opts.Protocol < RouteProtoInfo { route.mu.Unlock() route.closeConnection(RouteRemoved) continue @@ -906,15 +906,15 @@ func (s *Server) reloadClusterPermissions() { // that we now possibly allow with a change of Export permissions. route.sendInfo(infoJSON) // Now send SUB and UNSUB protocols as needed. - closed := route.sendRouteSubProtos(subsNeedSUB, nil) + closed := route.sendRouteSubProtos(subsNeedSUB, false, nil) if !closed { - route.sendRouteUnSubProtos(subsNeedUNSUB, nil) + route.sendRouteUnSubProtos(subsNeedUNSUB, false, nil) } route.mu.Unlock() } // Remove as a batch all the subs that we have removed from each route. // FIXME(dlc) - Change for accounts. - s.gsl.RemoveBatch(deleteRoutedSubs) + s.gacc.sl.RemoveBatch(deleteRoutedSubs) } // validateClusterOpts ensures the new ClusterOpts does not change host or diff --git a/server/reload_test.go b/server/reload_test.go index 03d0bc01..6e16eaeb 100644 --- a/server/reload_test.go +++ b/server/reload_test.go @@ -17,7 +17,6 @@ import ( "encoding/base64" "encoding/json" "fmt" - "github.com/nats-io/nkeys" "io/ioutil" "net" "os" @@ -28,6 +27,8 @@ import ( "testing" "time" + "github.com/nats-io/nkeys" + "github.com/nats-io/go-nats" ) @@ -1861,12 +1862,16 @@ func TestConfigReloadRotateFiles(t *testing.T) { server, _, config := runReloadServerWithConfig(t, "./configs/reload/file_rotate.conf") defer func() { os.Remove(config) + os.Remove("log.txt") + os.Remove("gnatsd.pid") os.Remove("log1.txt") os.Remove("gnatsd1.pid") }() defer server.Shutdown() // Configure the logger to enable actual logging + opts := server.getOpts() + opts.NoLog = false server.ConfigureLogger() // Load a config that renames the files. @@ -2379,11 +2384,11 @@ func TestConfigReloadClusterPermsOldServer(t *testing.T) { optsB := DefaultOptions() optsB.Routes = RoutesFromStr(fmt.Sprintf("nats://127.0.0.1:%d", srva.ClusterAddr().Port)) // Make server B behave like an old server - testRouteProto = routeProtoZero - defer func() { testRouteProto = routeProtoInfo }() + testRouteProto = RouteProtoZero + defer func() { testRouteProto = RouteProtoInfo }() srvb := RunServer(optsB) defer srvb.Shutdown() - testRouteProto = routeProtoInfo + testRouteProto = RouteProtoInfo checkClusterFormed(t, srva, srvb) @@ -2683,10 +2688,6 @@ func TestConfigReloadAccountNKeyUsers(t *testing.T) { if c.acc != synadia { t.Fatalf("Expected the nkey client's account to match 'synadia', got %v", c.acc) } - // Also test client sublist. - if c.sl != synadia.sl { - t.Fatalf("Expected the client's sublist to match 'synadia' account") - } // Now nats account nkey user. kp, _ = nkeys.FromSeed(seed2) @@ -2717,10 +2718,6 @@ func TestConfigReloadAccountNKeyUsers(t *testing.T) { if c.acc != nats { t.Fatalf("Expected the nkey client's account to match 'nats', got %v", c.acc) } - // Also test client sublist. - if c.sl != nats.sl { - t.Fatalf("Expected the client's sublist to match 'nats' account") - } // Remove user from account and whole account reloadUpdateConfig(t, s, conf, ` @@ -2860,7 +2857,7 @@ func TestConfigReloadAccountStreamsImportExport(t *testing.T) { t.Helper() dcli := s.getClient(1) dcli.mu.Lock() - r := dcli.sl.Match(subject) + r := dcli.acc.sl.Match(subject) dcli.mu.Unlock() if shouldBeThere && len(r.psubs) != 1 { t.Fatalf("%s should have 1 match in derek's sublist, got %v", subject, len(r.psubs)) diff --git a/server/route.go b/server/route.go index 7e82d627..8a99796a 100644 --- a/server/route.go +++ b/server/route.go @@ -14,7 +14,6 @@ package server import ( - "bytes" "crypto/tls" "encoding/json" "fmt" @@ -42,14 +41,16 @@ const ( const ( // routeProtoZero is the original Route protocol from 2009. // http://nats.io/documentation/internals/nats-protocol/ - routeProtoZero = iota + RouteProtoZero = iota // routeProtoInfo signals a route can receive more then the original INFO block. // This can be used to update remote cluster permissions, etc... - routeProtoInfo + RouteProtoInfo + // This is the new route/cluster protocol that provides account support. + RouteProtoV2 ) // Used by tests -var testRouteProto = routeProtoInfo +var testRouteProto = RouteProtoV2 type route struct { remoteID string @@ -61,6 +62,7 @@ type route struct { tlsRequired bool closed bool connectURLs []string + replySubs map[*subscription]*time.Timer } type connectInfo struct { @@ -73,197 +75,197 @@ type connectInfo struct { Name string `json:"name"` } -// Used to hold onto mappings for unsubscribed -// routed queue subscribers. -type rqsub struct { - group []byte - atime time.Time -} - // Route protocol constants const ( ConProto = "CONNECT %s" + _CRLF_ InfoProto = "INFO %s" + _CRLF_ ) -// Clear up the timer and any map held for remote qsubs. -func (s *Server) clearRemoteQSubs() { - s.rqsMu.Lock() - defer s.rqsMu.Unlock() - if s.rqsubsTimer != nil { - s.rqsubsTimer.Stop() - s.rqsubsTimer = nil +// This will add a timer to watch over remote reply subjects in case +// the fail to receive a response. The duration will be taken from the +// accounts map timeout to match. +// Lock should be held upon entering. +func (c *client) addReplySubTimeout(sub *subscription, d time.Duration) { + if c.route.replySubs == nil { + c.route.replySubs = make(map[*subscription]*time.Timer) } - s.rqsubs = nil + rs := c.route.replySubs + rs[sub] = time.AfterFunc(d, func() { + c.mu.Lock() + delete(rs, sub) + sub.max = 0 + c.mu.Unlock() + c.unsubscribe(sub, true) + }) } -// Check to see if we can remove any of the remote qsubs mappings -func (s *Server) purgeRemoteQSubs() { - ri := s.getOpts().RQSubsSweep - s.rqsMu.Lock() - exp := time.Now().Add(-ri) - for k, rqsub := range s.rqsubs { - if exp.After(rqsub.atime) { - delete(s.rqsubs, k) - } - } - if s.rqsubsTimer != nil { - // Reset timer. - s.rqsubsTimer = time.AfterFunc(ri, s.purgeRemoteQSubs) - } - s.rqsMu.Unlock() -} - -// Lookup a remote queue group sid. -func (s *Server) lookupRemoteQGroup(sid string) []byte { - s.rqsMu.RLock() - rqsub := s.rqsubs[sid] - s.rqsMu.RUnlock() - return rqsub.group -} - -// This will hold onto a remote queue subscriber to allow -// for mapping and handling if we get a message after the -// subscription goes away. -func (s *Server) holdRemoteQSub(sub *subscription) { - // Should not happen, but protect anyway. - if len(sub.queue) == 0 { +// removeReplySubTimeout will remove a timer if it exists. +// Lock should be held upon entering. +func (c *client) removeReplySubTimeout(sub *subscription) { + // Remove any reply sub timer if it exists. + if c.route.replySubs == nil { return } - // Add the entry - s.rqsMu.Lock() - // Start timer if needed. - if s.rqsubsTimer == nil { - ri := s.getOpts().RQSubsSweep - s.rqsubsTimer = time.AfterFunc(ri, s.purgeRemoteQSubs) + if t, ok := c.route.replySubs[sub]; ok { + t.Stop() + delete(c.route.replySubs, sub) } - // Create map if needed. - if s.rqsubs == nil { - s.rqsubs = make(map[string]rqsub) - } - group := make([]byte, len(sub.queue)) - copy(group, sub.queue) - rqsub := rqsub{group: group, atime: time.Now()} - s.rqsubs[routeSid(sub)] = rqsub - s.rqsMu.Unlock() } -// This is for when we receive a directed message for a queue subscriber -// that has gone away. We reroute like a new message but scope to only -// the queue subscribers that it was originally intended for. We will -// prefer local clients, but will bounce to another route if needed. -func (c *client) reRouteQMsg(r *SublistResult, msgh, msg, group []byte) { - c.Debugf("Attempting redelivery of message for absent queue subscriber on group '%q'", group) +func (c *client) processAccountSub(arg []byte) error { + // Placeholder in case we add in to the protocol active senders of + // informtation. For now we do not do account interest propagation. + c.traceInOp("A+", arg) + return nil +} - // We only care about qsubs here. Data structure not setup for optimized - // lookup for our specific group however. +func (c *client) processAccountUnsub(arg []byte) { + // Placeholder in case we add in to the protocol active senders of + // informtation. For now we do not do account interest propagation. + c.traceInOp("AUSUB", arg) +} - var qsubs []*subscription - for _, qs := range r.qsubs { - if len(qs) != 0 && bytes.Equal(group, qs[0].queue) { - qsubs = qs - break - } +// Process an inbound RMSG specification from the remote route. +func (c *client) processRoutedMsgArgs(arg []byte) error { + if c.trace { + c.traceInOp("RMSG", arg) } - // If no match return. - if qsubs == nil { - c.Debugf("Redelivery failed, no queue subscribers for message on group '%q'", group) + // Unroll splitArgs to avoid runtime/heap issues + a := [MAX_MSG_ARGS][]byte{} + args := a[:0] + start := -1 + for i, b := range arg { + switch b { + case ' ', '\t', '\r', '\n': + if start >= 0 { + args = append(args, arg[start:i]) + start = -1 + } + default: + if start < 0 { + start = i + } + } + } + if start >= 0 { + args = append(args, arg[start:]) + } + + c.pa.arg = arg + switch len(args) { + case 0, 1, 2: + return fmt.Errorf("processRoutedMsgArgs Parse Error: '%s'", args) + case 3: + c.pa.reply = nil + c.pa.queues = nil + c.pa.szb = args[2] + c.pa.size = parseSize(args[2]) + case 4: + c.pa.reply = args[2] + c.pa.queues = nil + c.pa.szb = args[3] + c.pa.size = parseSize(args[3]) + default: + // args[2] is our reply indicator. Should be + or | normally. + if len(args[2]) != 1 { + return fmt.Errorf("processRoutedMsgArgs Bad or Missing Reply Indicator: '%s'", args[2]) + } + switch args[2][0] { + case '+': + c.pa.reply = args[3] + case '|': + c.pa.reply = nil + default: + return fmt.Errorf("processRoutedMsgArgs Bad or Missing Reply Indicator: '%s'", args[2]) + } + // Grab size. + c.pa.szb = args[len(args)-1] + c.pa.size = parseSize(c.pa.szb) + + // Grab queue names. + if c.pa.reply != nil { + c.pa.queues = args[4 : len(args)-1] + } else { + c.pa.queues = args[3 : len(args)-1] + } + } + if c.pa.size < 0 { + return fmt.Errorf("processRoutedMsgArgs Bad or Missing Size: '%s'", args) + } + + // Common ones processed after check for arg length + c.pa.account = args[0] + c.pa.subject = args[1] + return nil +} + +// processInboundRouteMsg is called to process an inbound msg from a route. +func (c *client) processInboundRouteMsg(msg []byte) { + // Update statistics + // The msg includes the CR_LF, so pull back out for accounting. + c.in.msgs += 1 + c.in.bytes += len(msg) - LEN_CR_LF + + if c.trace { + c.traceMsg(msg) + } + + if c.opts.Verbose { + c.sendOK() + } + + // Mostly under testing scenarios. + if c.srv == nil { return } - // We have a matched group of queue subscribers. - // We prefer a local subscriber since that was the original target. - - // Spin prand if needed. - if c.in.prand == nil { - c.in.prand = rand.New(rand.NewSource(time.Now().UnixNano())) - } - - // Hold onto a remote if we come across it to utilize in case no locals exist. - var rsub *subscription - - startIndex := c.in.prand.Intn(len(qsubs)) - for i := 0; i < len(qsubs); i++ { - index := (startIndex + i) % len(qsubs) - sub := qsubs[index] - if sub == nil { - continue - } - if rsub == nil && bytes.HasPrefix(sub.sid, []byte(QRSID)) { - rsub = sub - continue - } - mh := c.msgHeader(msgh[:], sub, c.pa.reply) - if c.deliverMsg(sub, mh, msg) { - c.Debugf("Redelivery succeeded for message on group '%q'", group) - return - } - } - // If we are here we failed to find a local, see if we snapshotted a - // remote sub, and if so deliver to that. - if rsub != nil { - mh := c.msgHeader(msgh[:], rsub, c.pa.reply) - if c.deliverMsg(rsub, mh, msg) { - c.Debugf("Re-routing message on group '%q' to remote server", group) - return - } - } - c.Debugf("Redelivery failed, no queue subscribers for message on group '%q'", group) -} - -// processRoutedMsgResults processes messages inbound from a route. -func (c *client) processRoutedMsgResults(r *SublistResult, msg []byte) { - // Snapshot server. - srv := c.srv - - // msg header - msgh := c.msgb[:msgHeadProtoLen] - msgh = append(msgh, c.pa.subject...) - msgh = append(msgh, ' ') - si := len(msgh) - - // If we have a queue subscription, deliver direct - // since they are sent direct via L2 semantics over routes. - // If the match is a queue subscription, we will return from - // here regardless if we find a sub. - isq, sub, err := srv.routeSidQueueSubscriber(c.pa.sid) - if isq { - if err != nil { - // We got an invalid QRSID, so stop here - c.Errorf("Unable to deliver routed queue message: %v", err) - return - } - didDeliver := false - if sub != nil { - mh := c.msgHeader(msgh[:si], sub, c.pa.reply) - didDeliver = c.deliverMsg(sub, mh, msg) - } - if !didDeliver && c.srv != nil { - group := c.srv.lookupRemoteQGroup(string(c.pa.sid)) - c.reRouteQMsg(r, msgh, msg, group) - } + // Match correct account and sublist. + // We might want to make a local version to avoid any contention. + acc := c.srv.LookupAccount(string(c.pa.account)) + if acc == nil { + c.Debugf("Unknown account %q for routed message on subject: %q", c.pa.account, c.pa.subject) return } - // Normal pub/sub message here - // Loop over all normal subscriptions that match. - for _, sub := range r.psubs { - // Check if this is a send to a ROUTER, if so we ignore to - // enforce 1-hop semantics. - if sub.client.typ == ROUTER { - continue - } - sub.client.mu.Lock() - if sub.client.nc == nil { - sub.client.mu.Unlock() - continue - } - sub.client.mu.Unlock() - // Normal delivery - mh := c.msgHeader(msgh[:si], sub, c.pa.reply) - c.deliverMsg(sub, mh, msg) + // Check to see if we need to map/route to another account. + if acc.imports.services != nil { + c.checkForImportServices(acc, msg) } + + // No L1 right now for routes since they multiplex over multiple accounts. + r := acc.sl.Match(string(c.pa.subject)) + + // Check for no interest, short circuit if so. + // This is the fanout scale. + if len(r.psubs)+len(r.qsubs) == 0 { + return + } + + // Check to see if we have a routed message with a service reply. + if isServiceReply(c.pa.reply) && acc != nil { + // Need to add a sub here for local interest to send a response back + // to the originating server/requestor where it will be re-mapped. + sid := make([]byte, 0, len(acc.Name)+len(c.pa.reply)+1) + sid = append(sid, acc.Name...) + sid = append(sid, ' ') + sid = append(sid, c.pa.reply...) + // Copy off the reply since otherwise we are referencing a buffer that will be reused. + reply := make([]byte, len(c.pa.reply)) + copy(reply, c.pa.reply) + sub := &subscription{client: c, subject: reply, sid: sid, max: 1} + if err := acc.sl.Insert(sub); err != nil { + c.Errorf("Could not insert subscription: %v", err) + } else { + ttl := acc.AutoExpireTTL() + c.mu.Lock() + c.subs[string(sid)] = sub + c.addReplySubTimeout(sub, ttl) + c.mu.Unlock() + } + } + + c.processMsgResults(acc, r, msg, c.pa.subject, c.pa.reply) } // Lock should be held entering here. @@ -371,8 +373,10 @@ func (c *client) processRouteInfo(info *Info) { if added, sendInfo := s.addRoute(c, info); added { c.Debugf("Registering remote route %q", info.ID) - // Send our local subscriptions to this route. - s.sendLocalSubsToRoute(c) + + // Send our subs to the other side. + s.sendSubsToRoute(c) + // sendInfo will be false if the route that we just accepted // is the only route there is. if sendInfo { @@ -428,9 +432,10 @@ func (s *Server) updateRemoteRoutePerms(route *client, info *Info) { _localSubs [4096]*subscription localSubs = _localSubs[:0] ) - s.gsl.localSubs(&localSubs) + // FIXME(dlc) - Add account scoping. + s.gacc.sl.localSubs(&localSubs) - route.sendRouteSubProtos(localSubs, func(sub *subscription) bool { + route.sendRouteSubProtos(localSubs, false, func(sub *subscription) bool { subj := sub.subject // If the remote can now export but could not before, and this server can import this // subject, then send SUB protocol. @@ -574,20 +579,245 @@ func (c *client) setRoutePermissions(perms *RoutePermissions) { c.setPermissions(p) } -// This will send local subscription state to a new route connection. -// FIXME(dlc) - This could be a DOS or perf issue with many clients -// and large subscription space. Plus buffering in place not a good idea. -func (s *Server) sendLocalSubsToRoute(route *client) { - var raw [4096]*subscription - subs := raw[:0] +// Type used to hold a list of subs on a per account basis. +type asubs struct { + acc *Account + subs []*subscription +} - // FIXME(dlc) this needs to be scoped per account when cluster proto changes. - s.gsl.localSubs(&subs) +// removeRemoteSubs will walk the subs and remove them from the appropriate account. +func (c *client) removeRemoteSubs() { + // We need to gather these on a per account basis. + // FIXME(dlc) - We should be smarter about this.. + as := map[string]*asubs{} + c.mu.Lock() + srv := c.srv + subs := c.subs + c.subs = make(map[string]*subscription) + c.mu.Unlock() + + for key, sub := range subs { + c.mu.Lock() + sub.max = 0 + c.mu.Unlock() + // Grab the account + accountName := strings.Fields(key)[0] + ase := as[accountName] + if ase == nil { + acc := srv.LookupAccount(accountName) + if acc == nil { + continue + } + as[accountName] = &asubs{acc: acc, subs: []*subscription{sub}} + } else { + ase.subs = append(ase.subs, sub) + } + } + + // Now remove the subs by batch for each account sublist. + for _, ase := range as { + c.Debugf("Removing %d subscriptions for account %q", len(ase.subs), ase.acc.Name) + ase.acc.sl.RemoveBatch(ase.subs) + } +} + +// Indicates no more interest in the given account/subject for the remote side. +func (c *client) processRemoteUnsub(arg []byte) (err error) { + c.traceInOp("RS-", arg) + + // Indicate activity. + c.in.subs++ + + srv := c.srv + if srv == nil { + return nil + } + + args := splitArg(arg) + sub := &subscription{client: c} + + switch len(args) { + case 2: + sub.queue = nil + case 3: + sub.queue = args[2] + default: + return fmt.Errorf("processRemoteUnSub Parse Error: '%s'", arg) + } + sub.subject = args[1] + + // Lookup the account + accountName := string(args[0]) + acc := c.srv.LookupAccount(accountName) + if acc == nil { + c.Debugf("Unknown account %q for subject %q", accountName, sub.subject) + // Mark this account as not interested since we received a RS- and we + // do not have any record of it. + return nil + } + + c.mu.Lock() + if c.nc == nil { + c.mu.Unlock() + return nil + } + + // We store local subs by account and subject and optionally queue name. + // RS- will have the arg exactly as the key. + key := string(arg) + if sub, ok := c.subs[key]; ok { + delete(c.subs, key) + acc.sl.Remove(sub) + c.removeReplySubTimeout(sub) + } + c.mu.Unlock() + + if c.opts.Verbose { + c.sendOK() + } + return nil +} + +func (c *client) processRemoteSub(argo []byte) (err error) { + c.traceInOp("RS+", argo) + + // Indicate activity. + c.in.subs++ + + srv := c.srv + if srv == nil { + return nil + } + + // Copy so we do not reference a potentially large buffer + arg := make([]byte, len(argo)) + copy(arg, argo) + + args := splitArg(arg) + sub := &subscription{client: c} + + switch len(args) { + case 2: + sub.queue = nil + case 4: + sub.queue = args[2] + sub.qw = int32(parseSize(args[3])) + default: + return fmt.Errorf("processRemoteSub Parse Error: '%s'", arg) + } + sub.subject = args[1] + + // Lookup the account + // FIXME(dlc) - This may start having lots of contention? + accountName := string(args[0]) + acc := c.srv.LookupAccount(accountName) + if acc == nil { + if !srv.NewAccountsAllowed() { + c.Debugf("Unknown account %q for subject %q", accountName, sub.subject) + return nil + } else { + acc, _ = srv.LookupOrRegisterAccount(accountName) + } + } + + c.mu.Lock() + if c.nc == nil { + c.mu.Unlock() + return nil + } + + // Check permissions if applicable. + if !c.canExport(sub.subject) { + c.mu.Unlock() + c.Debugf("Can not export %q, ignoring remote subscription request", sub.subject) + return nil + } + + // Check if we have a maximum on the number of subscriptions. + if c.msubs > 0 && len(c.subs) >= c.msubs { + c.mu.Unlock() + c.maxSubsExceeded() + return nil + } + + // We store local subs by account and subject and optionally queue name. + // If we have a queue it will have a trailing weight which we do not want. + if sub.queue != nil { + sub.sid = arg[:len(arg)-len(args[3])-1] + } else { + sub.sid = arg + } + key := string(sub.sid) + osub := c.subs[key] + if osub == nil { + c.subs[string(key)] = sub + // Now place into the account sl. + if err = acc.sl.Insert(sub); err != nil { + delete(c.subs, key) + c.mu.Unlock() + c.Errorf("Could not insert subscription: %v", err) + c.sendErr("Invalid Subscription") + return nil + } + } else if sub.queue != nil { + // For a queue we need to update the weight. + atomic.StoreInt32(&osub.qw, sub.qw) + acc.sl.UpdateRemoteQSub(osub) + } + c.mu.Unlock() + + if c.opts.Verbose { + c.sendOK() + } + return nil +} + +// sendSubsToRoute will send over our subject interest to +// the remote side. For each account we will send the +// complete interest for all subjects, both normal as a binary +// and queue group weights. +func (s *Server) sendSubsToRoute(route *client) { + // Send over our account subscriptions. + var _accs [4096]*Account + accs := _accs[:0] + // copy accounts into array first + s.mu.Lock() + for _, a := range s.accounts { + accs = append(accs, a) + } + s.mu.Unlock() + + var raw [4096]*subscription + var closed bool route.mu.Lock() - closed := route.sendRouteSubProtos(subs, func(sub *subscription) bool { - return route.canImport(sub.subject) - }) + for _, a := range accs { + subs := raw[:0] + a.mu.RLock() + for key, rme := range a.rm { + // FIXME(dlc) - Just pass rme around. + // Construct a sub on the fly + var qn []byte + subEnd := len(key) + if qi := rme.qi; qi > 0 { + subEnd = int(qi) - 1 + qn = []byte(key[qi:]) + } + sub := &subscription{subject: []byte(key[:subEnd]), queue: qn, qw: rme.n} + subs = append(subs, sub) + + } + a.mu.RUnlock() + + closed = route.sendRouteSubProtos(subs, false, func(sub *subscription) bool { + return route.canImport(sub.subject) + }) + + if closed { + route.mu.Unlock() + return + } + } route.mu.Unlock() if !closed { route.Debugf("Sent local subscriptions to route") @@ -599,8 +829,8 @@ func (s *Server) sendLocalSubsToRoute(route *client) { // This function may release the route's lock due to flushing of outbound data. A boolean // is returned to indicate if the connection has been closed during this call. // Lock is held on entry. -func (c *client) sendRouteSubProtos(subs []*subscription, filter func(sub *subscription) bool) bool { - return c.sendRouteSubOrUnSubProtos(subs, true, filter) +func (c *client) sendRouteSubProtos(subs []*subscription, trace bool, filter func(sub *subscription) bool) bool { + return c.sendRouteSubOrUnSubProtos(subs, true, trace, filter) } // Sends UNSUBs protocols for the given subscriptions. If a filter is specified, it is @@ -608,14 +838,14 @@ func (c *client) sendRouteSubProtos(subs []*subscription, filter func(sub *subsc // This function may release the route's lock due to flushing of outbound data. A boolean // is returned to indicate if the connection has been closed during this call. // Lock is held on entry. -func (c *client) sendRouteUnSubProtos(subs []*subscription, filter func(sub *subscription) bool) bool { - return c.sendRouteSubOrUnSubProtos(subs, false, filter) +func (c *client) sendRouteUnSubProtos(subs []*subscription, trace bool, filter func(sub *subscription) bool) bool { + return c.sendRouteSubOrUnSubProtos(subs, false, trace, filter) } // Low-level function that sends SUBs or UNSUBs protcols for the given subscriptions. // Use sendRouteSubProtos or sendRouteUnSubProtos instead for clarity. // Lock is held on entry. -func (c *client) sendRouteSubOrUnSubProtos(subs []*subscription, isSubProto bool, filter func(sub *subscription) bool) bool { +func (c *client) sendRouteSubOrUnSubProtos(subs []*subscription, isSubProto, trace bool, filter func(sub *subscription) bool) bool { const staticBufSize = maxBufSize * 2 var ( _buf [staticBufSize]byte // array on stack @@ -632,18 +862,22 @@ func (c *client) sendRouteSubOrUnSubProtos(subs []*subscription, isSubProto bool if filter != nil && !filter(sub) { continue } - rsid := routeSid(sub) + // Determine the account. If sub has an ImportMap entry, use that, otherwise scoped to + // client. Default to global if all else fails. + var accName string + if sub.im != nil { + accName = sub.im.acc.Name + } else if sub.client != nil && sub.client.acc != nil { + accName = sub.client.acc.Name + } else { + accName = globalAccountName + } + // Check if proto is going to fit. curSize := len(buf) - if isSubProto { - // "SUB " + subject + " " + queue + " " + ... - curSize += 4 + len(sub.subject) + 1 + len(sub.queue) + 1 - } else { - // "UNSUB " + ... - curSize += 6 - } - // rsid + "\r\n" - curSize += len(rsid) + 2 + // "RS+/- " account + subject + " " + queue + " " + account + curSize += 4 + len(accName) + len(sub.subject) + 1 + len(sub.queue) + 2 + if curSize >= mbs { if c.queueOutbound(buf) { // Need to allocate new array @@ -661,18 +895,33 @@ func (c *client) sendRouteSubOrUnSubProtos(subs []*subscription, isSubProto bool break } } + as := len(buf) if isSubProto { - buf = append(buf, []byte("SUB ")...) - buf = append(buf, sub.subject...) - buf = append(buf, ' ') - if len(sub.queue) > 0 { - buf = append(buf, sub.queue...) - } + buf = append(buf, []byte("RS+ ")...) } else { - buf = append(buf, []byte("UNSUB ")...) + buf = append(buf, []byte("RS- ")...) } + buf = append(buf, []byte(accName)...) buf = append(buf, ' ') - buf = append(buf, rsid...) + buf = append(buf, sub.subject...) + if len(sub.queue) > 0 { + buf = append(buf, ' ') + buf = append(buf, sub.queue...) + // Send our weight if we are a sub proto + if isSubProto { + buf = append(buf, ' ') + var b [12]byte + var i = len(b) + for l := sub.qw; l > 0; l /= 10 { + i -= 1 + b[i] = digits[l%10] + } + buf = append(buf, b[i:]...) + } + } + if trace { + c.traceOutOp("", buf[as:]) + } buf = append(buf, []byte(CR_LF)...) } if !closed && len(buf) > 0 { @@ -695,10 +944,11 @@ func (s *Server) createRoute(conn net.Conn, rURL *url.URL) *client { } } - c := &client{srv: s, sl: s.gsl, nc: conn, opts: clientOpts{}, typ: ROUTER, route: r} + c := &client{srv: s, nc: conn, opts: clientOpts{}, typ: ROUTER, route: r} // Grab server variables s.mu.Lock() + s.generateRouteInfoJSON() infoJSON := s.routeInfoJSON authRequired := s.routeInfo.AuthRequired tlsRequired := s.routeInfo.TLSRequired @@ -710,6 +960,9 @@ func (s *Server) createRoute(conn net.Conn, rURL *url.URL) *client { // Initialize c.initClient() + // Remember numAcccounts we are sending to the other side since things can change + // and we want to be deterministic about who sends first account list. + if didSolicit { // Do this before the TLS code, otherwise, in case of failure // and if route is explicit, it would try to reconnect to 'nil'... @@ -814,8 +1067,8 @@ func (s *Server) createRoute(conn net.Conn, rURL *url.URL) *client { } // Send our info to the other side. + // Our new version requires dynamic information for accounts and a nonce. c.sendInfo(infoJSON) - c.mu.Unlock() c.Noticef("Route connection created") @@ -827,97 +1080,6 @@ const ( _EMPTY_ = "" ) -const ( - subProto = "SUB %s %s %s" + _CRLF_ - unsubProto = "UNSUB %s" + _CRLF_ -) - -// FIXME(dlc) - Make these reserved and reject if they come in as a sid -// from a client connection. -// Route constants -const ( - RSID = "RSID" - QRSID = "QRSID" - - QRSID_LEN = len(QRSID) -) - -// Parse the given rsid. If the protocol does not start with QRSID, -// returns false and no subscription nor error. -// If it does start with QRSID, returns true and possibly a subscription -// or an error if the QRSID protocol is malformed. -func (s *Server) routeSidQueueSubscriber(rsid []byte) (bool, *subscription, error) { - if !bytes.HasPrefix(rsid, []byte(QRSID)) { - return false, nil, nil - } - cid, sid, err := parseRouteQueueSid(rsid) - if err != nil { - return true, nil, err - } - - s.mu.Lock() - client := s.clients[cid] - s.mu.Unlock() - - if client == nil { - return true, nil, nil - } - - client.mu.Lock() - sub, ok := client.subs[string(sid)] - client.mu.Unlock() - if ok { - return true, sub, nil - } - return true, nil, nil -} - -// Creates a routable sid that can be used -// to reach remote subscriptions. -func routeSid(sub *subscription) string { - var qi string - if len(sub.queue) > 0 { - qi = "Q" - } - return fmt.Sprintf("%s%s:%d:%s", qi, RSID, sub.client.cid, sub.sid) -} - -// Parse the given `rsid` knowing that it starts with `QRSID`. -// Returns the cid and sid or an error not a valid QRSID. -func parseRouteQueueSid(rsid []byte) (uint64, []byte, error) { - var ( - cid uint64 - sid []byte - cidFound bool - sidFound bool - ) - // A valid QRSID needs to be at least QRSID:x:y - // First character here should be `:` - if len(rsid) >= QRSID_LEN+4 { - if rsid[QRSID_LEN] == ':' { - for i, count := QRSID_LEN+1, len(rsid); i < count; i++ { - switch rsid[i] { - case ':': - cid = uint64(parseInt64(rsid[QRSID_LEN+1 : i])) - cidFound = true - sid = rsid[i+1:] - } - } - if cidFound { - // We can't assume the content of sid, so as long - // as it is not len 0, we have to say it is a valid one. - if len(rsid) > 0 { - sidFound = true - } - } - } - } - if cidFound && sidFound { - return cid, sid, nil - } - return 0, nil, fmt.Errorf("invalid QRSID: %s", rsid) -} - func (s *Server) addRoute(c *client, info *Info) (bool, bool) { id := c.route.remoteID sendInfo := false @@ -976,83 +1138,123 @@ func (s *Server) addRoute(c *client, info *Info) (bool, bool) { return !exists, sendInfo } -func (s *Server) broadcastInterestToRoutes(sub *subscription, proto string) { - var arg []byte - if atomic.LoadInt32(&s.logging.trace) == 1 { - arg = []byte(proto[:len(proto)-LEN_CR_LF]) +// updateRouteSubscriptionMap will make sure to update the route map for the subscription. Will +// also forward to all routes if needed. +func (s *Server) updateRouteSubscriptionMap(acc *Account, sub *subscription, delta int32) { + if acc == nil || sub == nil { + return } - protoAsBytes := []byte(proto) - checkPerms := true - s.mu.Lock() - for _, route := range s.routes { - // FIXME(dlc) - Make same logic as deliverMsg - route.mu.Lock() - // The permission of this cluster applies to all routes, and each - // route will have the same `perms`, so check with the first route - // and send SUB interest only if subject has a match in import permissions. - // If there is no match, we stop here. - if checkPerms { - checkPerms = false - if !route.canImport(sub.subject) { - route.mu.Unlock() - break - } + acc.mu.RLock() + rm := acc.rm + acc.mu.RUnlock() + + // This is non-nil when we know we are in cluster mode. + if rm == nil { + return + } + + // We only store state on local subs for transmission across routes. + if sub.client == nil || sub.client.typ != CLIENT { + return + } + + // Create the fast key which will use the subject or 'subjectqueue' for queue subscribers. + var ( + _rkey [1024]byte + key []byte + qi int + ) + if sub.queue != nil { + // Just make the key subject spc group, e.g. 'foo bar' + key = _rkey[:0] + key = append(key, sub.subject...) + key = append(key, byte(' ')) + qi = len(key) + key = append(key, sub.queue...) + } else { + key = sub.subject + } + + // We always update for a queue subscriber since we need to send our relative weight. + var entry *rme + var ok bool + + // Always update if a queue subscriber. + update := qi > 0 + + // Copy to hold outside acc lock. + var entryN int32 + + acc.mu.Lock() + if entry, ok = rm[string(key)]; ok { + entry.n += delta + if entry.n <= 0 { + delete(rm, string(key)) + update = true // Update for deleting, } - route.sendProto(protoAsBytes, true) - route.mu.Unlock() - route.traceOutOp("", arg) + } else if delta > 0 { + entry = &rme{qi, delta} + rm[string(key)] = entry + update = true // Adding for normal sub means update. + } + if entry != nil { + entryN = entry.n + } + acc.mu.Unlock() + + if !update || entry == nil { + return + } + // We need to send out this update. + + // If we are sending a queue sub, copy and place in the queue weight. + if sub.queue != nil { + sub.client.mu.Lock() + nsub := *sub + sub.client.mu.Unlock() + nsub.qw = entryN + sub = &nsub + } + + // Note that queue unsubs where entry.n > 0 are still + // subscribes with a smaller weight. + if entryN > 0 { + s.broadcastSubscribe(sub) + } else { + s.broadcastUnSubscribe(sub) } - s.mu.Unlock() } // broadcastSubscribe will forward a client subscription -// to all active routes. +// to all active routes as needed. func (s *Server) broadcastSubscribe(sub *subscription) { - if s.numRoutes() == 0 { - return + trace := atomic.LoadInt32(&s.logging.trace) == 1 + s.mu.Lock() + subs := []*subscription{sub} + for _, route := range s.routes { + route.mu.Lock() + route.sendRouteSubProtos(subs, trace, func(sub *subscription) bool { + return route.canImport(sub.subject) + }) + route.mu.Unlock() } - rsid := routeSid(sub) - proto := fmt.Sprintf(subProto, sub.subject, sub.queue, rsid) - s.broadcastInterestToRoutes(sub, proto) + s.mu.Unlock() } // broadcastUnSubscribe will forward a client unsubscribe // action to all active routes. func (s *Server) broadcastUnSubscribe(sub *subscription) { - if s.numRoutes() == 0 { - return - } - sub.client.mu.Lock() - // Max has no meaning on the other side of a route, so do not send. - hasMax := sub.max > 0 && sub.nm < sub.max - sub.client.mu.Unlock() - if hasMax { - return - } - rsid := routeSid(sub) - proto := fmt.Sprintf(unsubProto, rsid) - s.broadcastInterestToRoutes(sub, proto) -} - -// Sends UNSUB protocols for each of the subscriptions in the given array -// to all connected routes. Used when a client connection is closed. Note -// that when that happens, the subscriptions' MAX have been cleared (force unsub). -func (s *Server) broadcastUnSubscribeBatch(subs []*subscription) { - var ( - _routes [32]*client - routes = _routes[:0] - ) + trace := atomic.LoadInt32(&s.logging.trace) == 1 s.mu.Lock() + subs := []*subscription{sub} for _, route := range s.routes { - routes = append(routes, route) - } - s.mu.Unlock() - - for _, route := range routes { route.mu.Lock() - route.sendRouteUnSubProtos(subs, nil) + route.sendRouteUnSubProtos(subs, trace, func(sub *subscription) bool { + return route.canImport(sub.subject) + }) route.mu.Unlock() } + s.mu.Unlock() } func (s *Server) routeAcceptLoop(ch chan struct{}) { @@ -1083,7 +1285,7 @@ func (s *Server) routeAcceptLoop(ch chan struct{}) { s.mu.Lock() // For tests, we want to be able to make this server behave - // as an older server. + // as an older server so we use the variable which we can override. proto := testRouteProto // Check for TLSConfig tlsReq := opts.Cluster.TLSConfig != nil @@ -1281,9 +1483,3 @@ func (s *Server) solicitRoutes(routes []*url.URL) { s.startGoRoutine(func() { s.connectToRoute(route, true) }) } } - -func (s *Server) numRoutes() int { - s.mu.Lock() - defer s.mu.Unlock() - return len(s.routes) -} diff --git a/server/routes_test.go b/server/routes_test.go index e84a54cf..57f6d3e3 100644 --- a/server/routes_test.go +++ b/server/routes_test.go @@ -863,7 +863,6 @@ func TestServerPoolUpdatedWhenRouteGoesAway(t *testing.T) { func TestRoutedQueueAutoUnsubscribe(t *testing.T) { optsA, _ := ProcessConfigFile("./configs/seed.conf") optsA.NoSigs, optsA.NoLog = true, true - optsA.RQSubsSweep = 500 * time.Millisecond srvA := RunServer(optsA) defer srvA.Shutdown() @@ -919,9 +918,22 @@ func TestRoutedQueueAutoUnsubscribe(t *testing.T) { t.Fatalf("Error on auto-unsubscribe: %v", err) } } - c.Flush() + c.Subscribe("TEST.COMPLETE", func(m *nats.Msg) {}) } + // We coelasce now so for each server we will have all local (250) plus + // two from the remote side for each queue group. We also create one more + // and will wait til each server has 254 subscriptions, that will make sure + // that we have everything setup. + checkFor(t, 10*time.Second, 100*time.Millisecond, func() error { + subsA := srvA.NumSubscriptions() + subsB := srvB.NumSubscriptions() + if subsA != 254 || subsB != 254 { + return fmt.Errorf("Not all subs processed yet: %d and %d", subsA, subsB) + } + return nil + }) + expected := int32(250) // Now send messages from each server for i := int32(0); i < expected; i++ { @@ -937,17 +949,6 @@ func TestRoutedQueueAutoUnsubscribe(t *testing.T) { nbaz := atomic.LoadInt32(&rbaz) if nbar == expected && nbaz == expected { time.Sleep(500 * time.Millisecond) - // Now check all mappings are gone. - srvA.rqsMu.RLock() - nrqsa := len(srvA.rqsubs) - srvA.rqsMu.RUnlock() - srvB.rqsMu.RLock() - nrqsb := len(srvB.rqsubs) - srvB.rqsMu.RUnlock() - if nrqsa != 0 || nrqsb != 0 { - return fmt.Errorf("Expected rqs mappings to have cleared, but got A:%d, B:%d", - nrqsa, nrqsb) - } return nil } return fmt.Errorf("Did not receive all %d queue messages, received %d for 'bar' and %d for 'baz'", @@ -1106,7 +1107,8 @@ func TestRouteSendLocalSubsWithLowMaxPending(t *testing.T) { defer nc.Close() numSubs := 1000 for i := 0; i < numSubs; i++ { - nc.Subscribe("foo.bar", func(_ *nats.Msg) {}) + subj := fmt.Sprintf("fo.bar.%d", i) + nc.Subscribe(subj, func(_ *nats.Msg) {}) } checkExpectedSubs(t, numSubs, srvA) diff --git a/server/server.go b/server/server.go index 8b3d09ef..b994a17f 100644 --- a/server/server.go +++ b/server/server.go @@ -66,39 +66,35 @@ type Info struct { type Server struct { gcid uint64 stats - mu sync.Mutex - prand *rand.Rand - info Info - configFile string - optsMu sync.RWMutex - opts *Options - running bool - shutdown bool - listener net.Listener - gsl *Sublist - accounts map[string]*Account - clients map[uint64]*client - routes map[uint64]*client - remotes map[string]*client - users map[string]*User - nkeys map[string]*NkeyUser - totalClients uint64 - closed *closedRingBuffer - done chan bool - start time.Time - http net.Listener - httpHandler http.Handler - profiler net.Listener - httpReqStats map[string]uint64 - routeListener net.Listener - routeInfo Info - routeInfoJSON []byte - quitCh chan struct{} - - // Tracking for remote QRSID tags. - rqsMu sync.RWMutex - rqsubs map[string]rqsub - rqsubsTimer *time.Timer + mu sync.Mutex + prand *rand.Rand + info Info + configFile string + optsMu sync.RWMutex + opts *Options + running bool + shutdown bool + listener net.Listener + gacc *Account + accounts map[string]*Account + activeAccounts int + clients map[uint64]*client + routes map[uint64]*client + remotes map[string]*client + users map[string]*User + nkeys map[string]*NkeyUser + totalClients uint64 + closed *closedRingBuffer + done chan bool + start time.Time + http net.Listener + httpHandler http.Handler + profiler net.Listener + httpReqStats map[string]uint64 + routeListener net.Listener + routeInfo Info + routeInfoJSON []byte + quitCh chan struct{} // Tracking Go routines grMu sync.Mutex @@ -178,7 +174,6 @@ func New(opts *Options) *Server { configFile: opts.ConfigFile, info: info, prand: rand.New(rand.NewSource(time.Now().UnixNano())), - gsl: NewSublist(), opts: opts, done: make(chan bool, 1), start: now, @@ -201,7 +196,8 @@ func New(opts *Options) *Server { s.accounts = make(map[string]*Account) // Create global account. - s.registerAccount(&Account{Name: globalAccountName, sl: s.gsl}) + s.gacc = &Account{Name: globalAccountName} + s.registerAccount(s.gacc) // For tracking clients s.clients = make(map[uint64]*client) @@ -254,6 +250,11 @@ func (s *Server) configureAccounts() { } func (s *Server) generateRouteInfoJSON() { + // New proto wants a nonce. + var raw [nonceLen]byte + nonce := raw[:] + s.generateNonce(nonce) + s.routeInfo.Nonce = string(nonce) b, _ := json.Marshal(s.routeInfo) pcs := [][]byte{[]byte("INFO"), b, []byte(CR_LF)} s.routeInfoJSON = bytes.Join(pcs, []byte(" ")) @@ -303,7 +304,7 @@ func (s *Server) logPid() error { } // newAccountsAllowed returns whether or not new accounts can be created on the fly. -func (s *Server) newAccountsAllowed() bool { +func (s *Server) NewAccountsAllowed() bool { s.mu.Lock() defer s.mu.Unlock() return s.opts.AllowNewAccounts @@ -315,6 +316,13 @@ func (s *Server) numReservedAccounts() int { return 1 } +// NumActiveAccounts reports number of active accounts on this server. +func (s *Server) NumActiveAccounts() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.activeAccounts +} + // LookupOrRegisterAccount will return the given account if known or create a new entry. func (s *Server) LookupOrRegisterAccount(name string) (account *Account, isNew bool) { s.mu.Lock() @@ -326,7 +334,7 @@ func (s *Server) LookupOrRegisterAccount(name string) (account *Account, isNew b Name: name, sl: NewSublist(), } - s.accounts[name] = acc + s.registerAccount(acc) return acc, true } @@ -355,6 +363,11 @@ func (s *Server) registerAccount(acc *Account) { if acc.maxaettl == 0 { acc.maxaettl = DEFAULT_TTL_AE_RESPONSE_MAP } + // If we are capable of routing we will track subscription + // information for efficient interest propagation. + if s.opts != nil && s.opts.Cluster.Port != 0 { + acc.rm = make(map[string]*rme, 256) + } s.accounts[acc.Name] = acc } @@ -497,8 +510,6 @@ func (s *Server) Shutdown() { s.profiler.Close() } - // Clear any remote qsub mappings - s.clearRemoteQSubs() s.mu.Unlock() // Release go routines that wait on that channel @@ -870,7 +881,9 @@ func (s *Server) createClient(conn net.Conn) *client { max_subs := opts.MaxSubs now := time.Now() - c := &client{srv: s, sl: s.gsl, nc: conn, opts: defaultOpts, mpay: max_pay, msubs: max_subs, start: now, last: now} + c := &client{srv: s, nc: conn, opts: defaultOpts, mpay: max_pay, msubs: max_subs, start: now, last: now} + + c.registerWithAccount(s.gacc) // Grab JSON info string s.mu.Lock() @@ -1180,14 +1193,14 @@ func (s *Server) getClient(cid uint64) *client { // NumSubscriptions will report how many subscriptions are active. func (s *Server) NumSubscriptions() uint32 { s.mu.Lock() - var subs uint32 + var subs int for _, acc := range s.accounts { if acc.sl != nil { - subs += acc.sl.Count() + subs += acc.TotalSubs() } } s.mu.Unlock() - return subs + return uint32(subs) } // NumSlowConsumers will report the number of slow consumers. diff --git a/server/split_test.go b/server/split_test.go index ff36c80e..ef8e45a2 100644 --- a/server/split_test.go +++ b/server/split_test.go @@ -24,8 +24,9 @@ func TestSplitBufferSubOp(t *testing.T) { defer cli.Close() defer trash.Close() - s := &Server{gsl: NewSublist()} - c := &client{srv: s, sl: s.gsl, subs: make(map[string]*subscription), nc: cli} + s := &Server{gacc: &Account{Name: globalAccountName}, accounts: make(map[string]*Account)} + s.registerAccount(s.gacc) + c := &client{srv: s, acc: s.gacc, subs: make(map[string]*subscription), nc: cli} subop := []byte("SUB foo 1\r\n") subop1 := subop[:6] @@ -43,7 +44,7 @@ func TestSplitBufferSubOp(t *testing.T) { if c.state != OP_START { t.Fatalf("Expected OP_START state vs %d\n", c.state) } - r := s.gsl.Match("foo") + r := s.gacc.sl.Match("foo") if r == nil || len(r.psubs) != 1 { t.Fatalf("Did not match subscription properly: %+v\n", r) } @@ -60,8 +61,9 @@ func TestSplitBufferSubOp(t *testing.T) { } func TestSplitBufferUnsubOp(t *testing.T) { - s := &Server{gsl: NewSublist()} - c := &client{srv: s, subs: make(map[string]*subscription)} + s := &Server{gacc: &Account{Name: globalAccountName}, accounts: make(map[string]*Account)} + s.registerAccount(s.gacc) + c := &client{srv: s, acc: s.gacc, subs: make(map[string]*subscription)} subop := []byte("SUB foo 1024\r\n") if err := c.parse(subop); err != nil { @@ -87,7 +89,7 @@ func TestSplitBufferUnsubOp(t *testing.T) { if c.state != OP_START { t.Fatalf("Expected OP_START state vs %d\n", c.state) } - r := s.gsl.Match("foo") + r := s.gacc.sl.Match("foo") if r != nil && len(r.psubs) != 0 { t.Fatalf("Should be no subscriptions in results: %+v\n", r) } @@ -300,7 +302,7 @@ func TestSplitConnectArg(t *testing.T) { func TestSplitDanglingArgBuf(t *testing.T) { s := New(&defaultServerOptions) - c := &client{srv: s, sl: s.gsl, subs: make(map[string]*subscription)} + c := &client{srv: s, acc: s.gacc, subs: make(map[string]*subscription)} // We test to make sure we do not dangle any argBufs after processing // since that could lead to performance issues. @@ -360,14 +362,14 @@ func TestSplitDanglingArgBuf(t *testing.T) { // MSG (the client has to be a ROUTE) c = &client{subs: make(map[string]*subscription), typ: ROUTER} - msgop := []byte("MSG foo RSID:2:1 5\r\nhello\r\n") + msgop := []byte("RMSG $foo foo 5\r\nhello\r\n") c.parse(msgop[:5]) c.parse(msgop[5:10]) if c.argBuf == nil { t.Fatal("Expected a non-nil argBuf") } - if string(c.argBuf) != "foo RS" { - t.Fatalf("Expected argBuf to be \"foo 1 \", got %q", string(c.argBuf)) + if string(c.argBuf) != "$foo " { + t.Fatalf("Expected argBuf to be \"$foo \", got %q", string(c.argBuf)) } c.parse(msgop[10:]) if c.argBuf != nil { @@ -384,21 +386,21 @@ func TestSplitDanglingArgBuf(t *testing.T) { if c.argBuf == nil { t.Fatal("Expected a non-nil argBuf") } + if string(c.pa.account) != "$foo" { + t.Fatalf("Expected account to be \"$foo\", got %q", c.pa.account) + } if string(c.pa.subject) != "foo" { t.Fatalf("Expected subject to be \"foo\", got %q", c.pa.subject) } if string(c.pa.reply) != "" { t.Fatalf("Expected reply to be \"\", got %q", c.pa.reply) } - if string(c.pa.sid) != "RSID:2:1" { - t.Fatalf("Expected sid to \"RSID:2:1\", got %q", c.pa.sid) - } if c.pa.size != 5 { t.Fatalf("Expected sid to 5, got %v", c.pa.size) } // msg buffer should be - if c.msgBuf == nil || string(c.msgBuf) != "hel" { - t.Fatalf("Expected msgBuf to be \"hel\", got %q", c.msgBuf) + if c.msgBuf == nil || string(c.msgBuf) != "hello\r" { + t.Fatalf("Expected msgBuf to be \"hello\r\", got %q", c.msgBuf) } c.parse(msgop[23:]) // At the end, we should have cleaned-up both arg and msg buffers. @@ -410,31 +412,29 @@ func TestSplitDanglingArgBuf(t *testing.T) { } } -func TestSplitMsgArg(t *testing.T) { +func TestSplitRoutedMsgArg(t *testing.T) { _, c, _ := setupClient() - // Allow parser to process MSG + // Allow parser to process RMSG c.typ = ROUTER b := make([]byte, 1024) - copy(b, []byte("MSG hello.world RSID:14:8 6040\r\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")) + copy(b, []byte("RMSG $G hello.world 6040\r\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")) c.parse(b) copy(b, []byte("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB\r\n")) c.parse(b) + wantAccount := "$G" wantSubject := "hello.world" - wantSid := "RSID:14:8" wantSzb := "6040" + if string(c.pa.account) != wantAccount { + t.Fatalf("Incorrect account: want %q, got %q", wantAccount, c.pa.account) + } if string(c.pa.subject) != wantSubject { t.Fatalf("Incorrect subject: want %q, got %q", wantSubject, c.pa.subject) } - - if string(c.pa.sid) != wantSid { - t.Fatalf("Incorrect sid: want %q, got %q", wantSid, c.pa.sid) - } - if string(c.pa.szb) != wantSzb { t.Fatalf("Incorrect szb: want %q, got %q", wantSzb, c.pa.szb) } @@ -442,21 +442,21 @@ func TestSplitMsgArg(t *testing.T) { func TestSplitBufferMsgOp(t *testing.T) { c := &client{subs: make(map[string]*subscription), typ: ROUTER} - msg := []byte("MSG foo.bar QRSID:15:3 _INBOX.22 11\r\nhello world\r") + msg := []byte("RMSG $G foo.bar _INBOX.22 11\r\nhello world\r") msg1 := msg[:2] msg2 := msg[2:9] msg3 := msg[9:15] msg4 := msg[15:22] msg5 := msg[22:25] msg6 := msg[25:37] - msg7 := msg[37:42] - msg8 := msg[42:] + msg7 := msg[37:40] + msg8 := msg[40:] if err := c.parse(msg1); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) } - if c.state != OP_MS { - t.Fatalf("Expected OP_MS state vs %d\n", c.state) + if c.state != OP_M { + t.Fatalf("Expected OP_M state vs %d\n", c.state) } if err := c.parse(msg2); err != nil { t.Fatalf("Unexpected parse error: %v\n", err) @@ -493,9 +493,6 @@ func TestSplitBufferMsgOp(t *testing.T) { if !bytes.Equal(c.pa.subject, []byte("foo.bar")) { t.Fatalf("MSG arg subject incorrect: '%s'\n", c.pa.subject) } - if !bytes.Equal(c.pa.sid, []byte("QRSID:15:3")) { - t.Fatalf("MSG arg sid incorrect: '%s'\n", c.pa.sid) - } if !bytes.Equal(c.pa.reply, []byte("_INBOX.22")) { t.Fatalf("MSG arg reply subject incorrect: '%s'\n", c.pa.reply) } diff --git a/server/sublist.go b/server/sublist.go index af553f47..5a7c914c 100644 --- a/server/sublist.go +++ b/server/sublist.go @@ -213,7 +213,7 @@ func (r *SublistResult) addSubToResult(sub *subscription) *SublistResult { if sub.queue == nil { nr.psubs = append(nr.psubs, sub) } else { - if i := findQSliceForSub(sub, nr.qsubs); i >= 0 { + if i := findQSlot(sub.queue, nr.qsubs); i >= 0 { nr.qsubs[i] = append(nr.qsubs[i], sub) } else { nr.qsubs = append(nr.qsubs, []*subscription{sub}) @@ -320,6 +320,23 @@ func (s *Sublist) reduceCacheCount() { }) } +// Helper function for auto-expanding remote qsubs. +func isRemoteQSub(sub *subscription) bool { + return sub != nil && sub.queue != nil && sub.client != nil && sub.client.typ == ROUTER +} + +// This should be called when we update the weight of an existing +// remote queue sub. +func (s *Sublist) UpdateRemoteQSub(sub *subscription) { + // We could search to make sure we find it, but probably not worth + // it unless we are thrashing the cache. Just remove from our L2 and update + // the genid so L1 will be flushed. + s.Lock() + s.removeFromCache(string(sub.subject), sub) + atomic.AddUint64(&s.genid, 1) + s.Unlock() +} + // This will add in a node's results to the total results. func addNodeToResults(n *node, results *SublistResult) { // Normal subscriptions @@ -335,18 +352,23 @@ func addNodeToResults(n *node, results *SublistResult) { if len(qr) == 0 { continue } - tsub := &subscription{subject: nil, queue: []byte(qname)} // Need to find matching list in results - if i := findQSliceForSub(tsub, results.qsubs); i >= 0 { - for _, sub := range qr { + var i int + if i = findQSlot([]byte(qname), results.qsubs); i < 0 { + i = len(results.qsubs) + nqsub := make([]*subscription, 0, len(qr)) + results.qsubs = append(results.qsubs, nqsub) + } + for _, sub := range qr { + if isRemoteQSub(sub) { + ns := atomic.LoadInt32(&sub.qw) + // Shadow these subscriptions + for n := 0; n < int(ns); n++ { + results.qsubs[i] = append(results.qsubs[i], sub) + } + } else { results.qsubs[i] = append(results.qsubs[i], sub) } - } else { - var nqsub []*subscription - for _, sub := range qr { - nqsub = append(nqsub, sub) - } - results.qsubs = append(results.qsubs, nqsub) } } } @@ -355,12 +377,12 @@ func addNodeToResults(n *node, results *SublistResult) { // processing publishes in L1 on client. So we need to walk sequentially // for now. Keep an eye on this in case we start getting large number of // different queue subscribers for the same subject. -func findQSliceForSub(sub *subscription, qsl [][]*subscription) int { - if sub.queue == nil { +func findQSlot(queue []byte, qsl [][]*subscription) int { + if queue == nil { return -1 } for i, qr := range qsl { - if len(qr) > 0 && bytes.Equal(sub.queue, qr[0].queue) { + if len(qr) > 0 && bytes.Equal(queue, qr[0].queue) { return i } } @@ -494,6 +516,56 @@ func (s *Sublist) RemoveBatch(subs []*subscription) error { return nil } +func (s *Sublist) checkNodeForClientSubs(n *node, c *client) { + var removed uint32 + for _, sub := range n.psubs { + if sub.client == c { + if s.removeFromNode(n, sub) { + s.removeFromCache(string(sub.subject), sub) + removed++ + } + } + } + // Queue subscriptions + for _, qr := range n.qsubs { + for _, sub := range qr { + if sub.client == c { + if s.removeFromNode(n, sub) { + s.removeFromCache(string(sub.subject), sub) + removed++ + } + } + } + } + s.count -= removed + s.removes += uint64(removed) +} + +func (s *Sublist) removeClientSubs(l *level, c *client) { + for _, n := range l.nodes { + s.checkNodeForClientSubs(n, c) + s.removeClientSubs(n.next, c) + } + if l.pwc != nil { + s.checkNodeForClientSubs(l.pwc, c) + s.removeClientSubs(l.pwc.next, c) + } + if l.fwc != nil { + s.checkNodeForClientSubs(l.fwc, c) + s.removeClientSubs(l.fwc.next, c) + } +} + +func (s *Sublist) RemoveAllForClient(c *client) { + s.Lock() + removes := s.removes + s.removeClientSubs(s.root, c) + if s.removes != removes { + atomic.AddUint64(&s.genid, 1) + } + s.Unlock() +} + // pruneNode is used to prune an empty node from the tree. func (l *level) pruneNode(n *node, t string) { if n == nil { @@ -541,7 +613,7 @@ func (s *Sublist) removeFromNode(n *node, sub *subscription) (found bool) { delete(n.psubs, sub) if found && n.plist != nil { // This will brute force remove the plist to perform - // correct behavior. Will get repopulated on a call + // correct behavior. Will get re-populated on a call // to Match as needed. n.plist = nil } @@ -549,12 +621,11 @@ func (s *Sublist) removeFromNode(n *node, sub *subscription) (found bool) { } // We have a queue group subscription here - qname := string(sub.queue) - qsub := n.qsubs[qname] + qsub := n.qsubs[string(sub.queue)] _, found = qsub[sub] delete(qsub, sub) if len(qsub) == 0 { - delete(n.qsubs, qname) + delete(n.qsubs, string(sub.queue)) } return found } @@ -830,7 +901,7 @@ func matchLiteral(literal, subject string) bool { } func addLocalSub(sub *subscription, subs *[]*subscription) { - if sub != nil && sub.client != nil && sub.client.typ == CLIENT { + if sub != nil && sub.client != nil && sub.client.typ == CLIENT && sub.im == nil { *subs = append(*subs, sub) } } @@ -855,11 +926,9 @@ func (s *Sublist) addNodeToSubs(n *node, subs *[]*subscription) { } func (s *Sublist) collectLocalSubs(l *level, subs *[]*subscription) { - if len(l.nodes) > 0 { - for _, n := range l.nodes { - s.addNodeToSubs(n, subs) - s.collectLocalSubs(n.next, subs) - } + for _, n := range l.nodes { + s.addNodeToSubs(n, subs) + s.collectLocalSubs(n.next, subs) } if l.pwc != nil { s.addNodeToSubs(l.pwc, subs) diff --git a/server/sublist_test.go b/server/sublist_test.go index 5801da3b..9829287d 100644 --- a/server/sublist_test.go +++ b/server/sublist_test.go @@ -20,6 +20,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "time" @@ -69,7 +70,7 @@ func verifyNumLevels(s *Sublist, expected int, t *testing.T) { } func verifyQMember(qsubs [][]*subscription, val *subscription, t *testing.T) { - verifyMember(qsubs[findQSliceForSub(val, qsubs)], val, t) + verifyMember(qsubs[findQSlot(val.queue, qsubs)], val, t) } func verifyMember(r []*subscription, val *subscription, t *testing.T) { @@ -86,7 +87,8 @@ func verifyMember(r []*subscription, val *subscription, t *testing.T) { // Helpers to generate test subscriptions. func newSub(subject string) *subscription { - return &subscription{subject: []byte(subject)} + c := &client{typ: CLIENT} + return &subscription{client: c, subject: []byte(subject)} } func newQSub(subject, queue string) *subscription { @@ -96,6 +98,14 @@ func newQSub(subject, queue string) *subscription { return newSub(subject) } +func newRemoteQSub(subject, queue string, num int32) *subscription { + if queue != "" { + c := &client{typ: ROUTER} + return &subscription{client: c, subject: []byte(subject), queue: []byte(queue), qw: num} + } + return newSub(subject) +} + func TestSublistInit(t *testing.T) { s := NewSublist() verifyCount(s, 0, t) @@ -247,6 +257,34 @@ func TestSublistRemoveWithLargeSubs(t *testing.T) { verifyLen(r.psubs, plistMin*2-3, t) } +func TestSublistRemoveByClient(t *testing.T) { + s := NewSublist() + c := &client{} + for i := 0; i < 10; i++ { + subject := fmt.Sprintf("a.b.c.d.e.f.%d", i) + sub := &subscription{client: c, subject: []byte(subject)} + s.Insert(sub) + } + verifyCount(s, 10, t) + s.Insert(&subscription{client: c, subject: []byte(">")}) + s.Insert(&subscription{client: c, subject: []byte("foo.*")}) + s.Insert(&subscription{client: c, subject: []byte("foo"), queue: []byte("bar")}) + s.Insert(&subscription{client: c, subject: []byte("foo"), queue: []byte("bar")}) + s.Insert(&subscription{client: c, subject: []byte("foo.bar"), queue: []byte("baz")}) + s.Insert(&subscription{client: c, subject: []byte("foo.bar"), queue: []byte("baz")}) + verifyCount(s, 16, t) + genid := atomic.LoadUint64(&s.genid) + s.RemoveAllForClient(c) + verifyCount(s, 0, t) + // genid should be different + if genid == atomic.LoadUint64(&s.genid) { + t.Fatalf("GenId should have been changed after removal of subs") + } + if cc := s.CacheCount(); cc != 0 { + t.Fatalf("Cache should be zero, got %d", cc) + } +} + func TestSublistInvalidSubjectsInsert(t *testing.T) { s := NewSublist() @@ -398,8 +436,8 @@ func TestSublistBasicQueueResults(t *testing.T) { r = s.Match(subject) verifyLen(r.psubs, 0, t) verifyQLen(r.qsubs, 2, t) - verifyLen(r.qsubs[findQSliceForSub(sub1, r.qsubs)], 1, t) - verifyLen(r.qsubs[findQSliceForSub(sub2, r.qsubs)], 2, t) + verifyLen(r.qsubs[findQSlot(sub1.queue, r.qsubs)], 1, t) + verifyLen(r.qsubs[findQSlot(sub2.queue, r.qsubs)], 2, t) verifyQMember(r.qsubs, sub2, t) verifyQMember(r.qsubs, sub3, t) verifyQMember(r.qsubs, sub4, t) @@ -757,6 +795,52 @@ func TestSublistRaceOnMatch(t *testing.T) { } } +// Remote subscriptions for queue subscribers will be weighted such that a single subscription +// is received, but represents all of the queue subscribers on the remote side. +func TestSublistRemoteQueueSubscriptions(t *testing.T) { + s := NewSublist() + // Normals + s1 := newQSub("foo", "bar") + s2 := newQSub("foo", "bar") + s.Insert(s1) + s.Insert(s2) + + // Now do weighted remotes. + rs1 := newRemoteQSub("foo", "bar", 10) + s.Insert(rs1) + rs2 := newRemoteQSub("foo", "bar", 10) + s.Insert(rs2) + + // These are just shadowed in results, so should appear as 4 subs. + verifyCount(s, 4, t) + + r := s.Match("foo") + verifyLen(r.psubs, 0, t) + verifyQLen(r.qsubs, 1, t) + verifyLen(r.qsubs[0], 22, t) + + s.Remove(s1) + s.Remove(rs1) + + verifyCount(s, 2, t) + + // Now make sure our shadowed results are correct after a removal. + r = s.Match("foo") + verifyLen(r.psubs, 0, t) + verifyQLen(r.qsubs, 1, t) + verifyLen(r.qsubs[0], 11, t) + + // Now do an update to an existing remote sub to update its weight. + rs2.qw = 1 + s.UpdateRemoteQSub(rs2) + + // Results should reflect new weight. + r = s.Match("foo") + verifyLen(r.psubs, 0, t) + verifyQLen(r.qsubs, 1, t) + verifyLen(r.qsubs[0], 2, t) +} + // -- Benchmarks Setup -- var subs []*subscription diff --git a/server/util.go b/server/util.go index 9fcfac6d..653b8ecd 100644 --- a/server/util.go +++ b/server/util.go @@ -23,12 +23,14 @@ import ( "strings" "time" - "github.com/nats-io/nuid" + "github.com/nats-io/nkeys" ) -// Use nuid. +// Use nkeys and the public key. func genID() string { - return nuid.Next() + kp, _ := nkeys.CreateServer() + pub, _ := kp.PublicKey() + return pub } // Ascii numbers 0-9 diff --git a/test/client_cluster_test.go b/test/client_cluster_test.go index a7ffc64e..36e89e34 100644 --- a/test/client_cluster_test.go +++ b/test/client_cluster_test.go @@ -128,7 +128,7 @@ func TestServerRestartAndQueueSubs(t *testing.T) { // Client options opts := nats.GetDefaultOptions() opts.Timeout = (5 * time.Second) - opts.ReconnectWait = (50 * time.Millisecond) + opts.ReconnectWait = (20 * time.Millisecond) opts.MaxReconnect = 1000 opts.NoRandomize = true @@ -267,7 +267,12 @@ func TestServerRestartAndQueueSubs(t *testing.T) { checkClusterFormed(t, srvA, srvB) // Make sure subscriptions are propagated in the cluster - if err := checkExpectedSubs(4, srvA, srvB); err != nil { + // Clients will be connected to srvA, so that will be 4, + // but srvB will only have 2 now since we coaelsce. + if err := checkExpectedSubs(4, srvA); err != nil { + t.Fatalf("%v", err) + } + if err := checkExpectedSubs(2, srvB); err != nil { t.Fatalf("%v", err) } diff --git a/test/cluster_test.go b/test/cluster_test.go index 4bca2fc3..4c431f8c 100644 --- a/test/cluster_test.go +++ b/test/cluster_test.go @@ -72,6 +72,15 @@ func checkExpectedSubs(expected int, servers ...*server.Server) error { return nil } +func runThreeServers(t *testing.T) (srvA, srvB, srvC *server.Server, optsA, optsB, optsC *server.Options) { + srvA, optsA = RunServerWithConfig("./configs/srv_a.conf") + srvB, optsB = RunServerWithConfig("./configs/srv_b.conf") + srvC, optsC = RunServerWithConfig("./configs/srv_c.conf") + + checkClusterFormed(t, srvA, srvB, srvC) + return +} + func runServers(t *testing.T) (srvA, srvB *server.Server, optsA, optsB *server.Options) { srvA, optsA = RunServerWithConfig("./configs/srv_a.conf") srvB, optsB = RunServerWithConfig("./configs/srv_b.conf") @@ -161,7 +170,8 @@ func TestClusterQueueSubs(t *testing.T) { expectA(pongRe) // Make sure the subs have propagated to srvB before continuing - if err := checkExpectedSubs(len(qg1SidsA), srvB); err != nil { + // New cluster proto this will only be 1. + if err := checkExpectedSubs(1, srvB); err != nil { t.Fatalf("%v", err) } @@ -188,7 +198,8 @@ func TestClusterQueueSubs(t *testing.T) { expectA(pongRe) // Make sure the subs have propagated to srvB before continuing - if err := checkExpectedSubs(len(qg1SidsA)+len(pSids), srvB); err != nil { + // Normal foo and the queue group will be one a piece, so 2 + wc == 3 + if err := checkExpectedSubs(3, srvB); err != nil { t.Fatalf("%v", err) } @@ -219,7 +230,8 @@ func TestClusterQueueSubs(t *testing.T) { expectB(pongRe) // Make sure the subs have propagated to srvA before continuing - if err := checkExpectedSubs(len(qg1SidsA)+len(pSids)+len(qg2SidsB), srvA); err != nil { + // This will be all the subs on A and just 1 from B that gets coalesced. + if err := checkExpectedSubs(len(qg1SidsA)+len(pSids)+1, srvA); err != nil { t.Fatalf("%v", err) } @@ -243,7 +255,7 @@ func TestClusterQueueSubs(t *testing.T) { expectA(pongRe) // Make sure the subs have propagated to srvB before continuing - if err := checkExpectedSubs(len(pSids)+len(qg2SidsB), srvB); err != nil { + if err := checkExpectedSubs(1+1+len(qg2SidsB), srvB); err != nil { t.Fatalf("%v", err) } @@ -302,7 +314,7 @@ func TestClusterDoubleMsgs(t *testing.T) { expectA1(pongRe) // Make sure the subs have propagated to srvB before continuing - if err := checkExpectedSubs(len(qg1SidsA), srvB); err != nil { + if err := checkExpectedSubs(1, srvB); err != nil { t.Fatalf("%v", err) } @@ -323,7 +335,7 @@ func TestClusterDoubleMsgs(t *testing.T) { pSids := []string{"1", "2"} // Make sure the subs have propagated to srvB before continuing - if err := checkExpectedSubs(len(qg1SidsA)+2, srvB); err != nil { + if err := checkExpectedSubs(1+2, srvB); err != nil { t.Fatalf("%v", err) } diff --git a/test/configs/new_cluster.conf b/test/configs/new_cluster.conf new file mode 100644 index 00000000..0d464cbe --- /dev/null +++ b/test/configs/new_cluster.conf @@ -0,0 +1,17 @@ +# New Cluster config file + +listen: 127.0.0.1:5343 + +cluster { + #nkey: CBSMNSOLVGFSP62Q2VD24KQIQXIVG2XVKSHE4DL7KKNN55MUYQKMDCHZ + listen: 127.0.0.1:5344 + + # Routes are actively solicited and connected to from this server. + # Other servers can connect to us if they supply the correct credentials + # in their routes definitions from above. + + routes = [ + nats-route://127.0.0.1:5345 + nats-route://127.0.0.1:5346 + ] +} diff --git a/test/configs/srv_c.conf b/test/configs/srv_c.conf new file mode 100644 index 00000000..534ceb8e --- /dev/null +++ b/test/configs/srv_c.conf @@ -0,0 +1,21 @@ +# Cluster Server C + +listen: 127.0.0.1:5226 + +cluster { + listen: 127.0.0.1:5248 + + authorization { + user: ruser + password: top_secret + timeout: 0.5 + } + + # Routes are actively solicited and connected to from this server. + # Other servers can connect to us if they supply the correct credentials + # in their routes definitions from above. + + routes = [ + nats-route://ruser:top_secret@127.0.0.1:5244 + ] +} diff --git a/test/new_routes_test.go b/test/new_routes_test.go new file mode 100644 index 00000000..3060fa27 --- /dev/null +++ b/test/new_routes_test.go @@ -0,0 +1,1227 @@ +// Copyright 2018 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/nats-io/gnatsd/server" + nats "github.com/nats-io/go-nats" +) + +func runNewRouteServer(t *testing.T) (*server.Server, *server.Options) { + return RunServerWithConfig("./configs/new_cluster.conf") +} + +func TestNewRouteInfoOnConnect(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + info := checkInfoMsg(t, rc) + if info.Port != opts.Cluster.Port { + t.Fatalf("Received wrong information for port, expected %d, got %d", + info.Port, opts.Cluster.Port) + } + + // Make sure we advertise new proto. + if info.Proto < server.RouteProtoV2 { + t.Fatalf("Expected routeProtoV2, got %d", info.Proto) + } + // New proto should always send nonce too. + if info.Nonce == "" { + t.Fatalf("Expected a non empty nonce in new route INFO") + } +} + +func TestNewRouteConnectSubs(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + + // Create 10 normal subs and 10 queue subscribers. + for i := 0; i < 10; i++ { + send(fmt.Sprintf("SUB foo %d\r\n", i)) + send(fmt.Sprintf("SUB foo bar %d\r\n", 100+i)) + } + send("PING\r\n") + expect(pongRe) + + // This client should not be considered active since no subscriptions or + // messages have been published. + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:22" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + info := checkInfoMsg(t, rc) + + // Send our info back with a larger number of accounts, should trigger to send the + // subscriptions for their accounts. + info.ID = routeID + b, err := json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + routeSend(fmt.Sprintf("INFO %s\r\n", b)) + + buf := routeExpect(rsubRe) + + matches := rsubRe.FindAllSubmatch(buf, -1) + if len(matches) != 2 { + t.Fatalf("Expected 2 results, got %d", len(matches)) + } + for _, m := range matches { + if string(m[1]) != "$G" { + t.Fatalf("Expected global account name of '$G', got %q", m[1]) + } + if string(m[2]) != "foo" { + t.Fatalf("Expected subject of 'foo', got %q", m[2]) + } + if m[3] != nil { + if string(m[3]) != "bar" { + t.Fatalf("Expected group of 'bar', got %q", m[3]) + } + // Expect the SID to be the total weighted count for the queue group + if len(m) != 5 { + t.Fatalf("Expected a SID for the queue group") + } + if m[4] == nil || string(m[4]) != "10" { + t.Fatalf("Expected SID of '10', got %q", m[4]) + } + } + } + + // Close the client connection, check the results. + c.Close() + + // Expect 2 + for numUnSubs := 0; numUnSubs != 2; { + buf := routeExpect(runsubRe) + numUnSubs += len(runsubRe.FindAllSubmatch(buf, -1)) + } +} + +func TestNewRouteRSubs(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + foo, err := s.RegisterAccount("$foo") + if err != nil { + t.Fatalf("Error creating account '$foo': %v", err) + } + bar, err := s.RegisterAccount("$bar") + if err != nil { + t.Fatalf("Error creating account '$bar': %v", err) + } + + // Create a client an account foo. + clientA := createClientConn(t, opts.Host, opts.Port) + sendA, expectA := setupConnWithAccount(t, clientA, "$foo") + defer clientA.Close() + sendA("PING\r\n") + expectA(pongRe) + + if foonc := foo.NumClients(); foonc != 1 { + t.Fatalf("Expected foo account to have 1 client, got %d", foonc) + } + if barnc := bar.NumClients(); barnc != 0 { + t.Fatalf("Expected bar account to have 0 clients, got %d", barnc) + } + + // Create a routeConn + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:33" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + info := checkInfoMsg(t, rc) + info.ID = routeID + b, err := json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + routeSend(fmt.Sprintf("INFO %s\r\nPING\r\n", b)) + routeExpect(pongRe) + + // Have the client listen on foo. + sendA("SUB foo 1\r\nPING\r\n") + expectA(pongRe) + + // Now create a new client for account $bar and have them subscribe. + clientB := createClientConn(t, opts.Host, opts.Port) + sendB, expectB := setupConnWithAccount(t, clientB, "$bar") + defer clientB.Close() + + sendB("PING\r\n") + expectB(pongRe) + + if foonc := foo.NumClients(); foonc != 1 { + t.Fatalf("Expected foo account to have 1 client, got %d", foonc) + } + if barnc := bar.NumClients(); barnc != 1 { + t.Fatalf("Expected bar account to have 1 client, got %d", barnc) + } + + // Have the client listen on foo. + sendB("SUB foo 1\r\nPING\r\n") + expectB(pongRe) + + routeExpect(rsubRe) + + // Unsubscribe on clientA from foo subject. + sendA("UNSUB 1\r\nPING\r\n") + expectA(pongRe) + + // We should get an RUSUB here. + routeExpect(runsubRe) + + // Now unsubscribe clientB, which should trigger an RS-. + sendB("UNSUB 1\r\nPING\r\n") + expectB(pongRe) + // We should get an RUSUB here. + routeExpect(runsubRe) + + // Now close down the clients. + clientA.Close() + + sendB("SUB foo 2\r\nPING\r\n") + expectB(pongRe) + + routeExpect(rsubRe) + + // Now close down client B. + clientB.Close() + + // This should trigger an RS- + routeExpect(runsubRe) +} + +func TestNewRouteProgressiveNormalSubs(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:33" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + info := checkInfoMsg(t, rc) + info.ID = routeID + b, err := json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + routeSend(fmt.Sprintf("INFO %s\r\n", b)) + routeSend("PING\r\n") + routeExpect(pongRe) + + // For progressive we will expect to receive first normal sub but + // not subsequent ones. + send("SUB foo 1\r\nPING\r\n") + expect(pongRe) + + routeExpect(rsubRe) + + send("SUB foo 2\r\nPING\r\n") + expect(pongRe) + expectNothing(t, rc) + + var buf []byte + + // Check that sid is showing us total number of subscriptions. + checkQueueSub := func(n string) { + matches := rsubRe.FindAllSubmatch(buf, -1) + if len(matches) != 1 { + t.Fatalf("Expected 1 result, got %d", len(matches)) + } + m := matches[0] + if len(m) != 5 { + t.Fatalf("Expected a SID for the queue group, only got %d elements", len(m)) + } + if string(m[4]) != n { + t.Fatalf("Expected %q, got %q", n, m[4]) + } + } + + // We should always get the SUB info for QUEUES. + send("SUB foo bar 3\r\nPING\r\n") + expect(pongRe) + buf = routeExpect(rsubRe) + checkQueueSub("1") + + send("SUB foo bar 4\r\nPING\r\n") + expect(pongRe) + buf = routeExpect(rsubRe) + checkQueueSub("2") + + send("SUB foo bar 5\r\nPING\r\n") + expect(pongRe) + buf = routeExpect(rsubRe) + checkQueueSub("3") + + // Now walk them back down. + // Again we should always get updates for queue subscribers. + // And these will be RS+ protos walking the weighted count back down. + send("UNSUB 5\r\nPING\r\n") + expect(pongRe) + buf = routeExpect(rsubRe) + checkQueueSub("2") + + send("UNSUB 4\r\nPING\r\n") + expect(pongRe) + buf = routeExpect(rsubRe) + checkQueueSub("1") + + // This one should send UNSUB + send("UNSUB 3\r\nPING\r\n") + expect(pongRe) + routeExpect(runsubRe) + + // Now normal ones. + send("UNSUB 1\r\nPING\r\n") + expect(pongRe) + expectNothing(t, rc) + + send("UNSUB 2\r\nPING\r\n") + expect(pongRe) + routeExpect(runsubRe) +} + +func TestNewRouteClientClosedWithNormalSubscriptions(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:44" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + info := checkInfoMsg(t, rc) + info.ID = routeID + b, err := json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + routeSend(fmt.Sprintf("INFO %s\r\n", b)) + + routeSend("PING\r\n") + routeExpect(pongRe) + + send("SUB foo 1\r\nPING\r\n") + expect(pongRe) + routeExpect(rsubRe) + + for i := 2; i < 100; i++ { + send(fmt.Sprintf("SUB foo %d\r\n", i)) + } + send("PING\r\n") + expect(pongRe) + + // Expect nothing from the route. + expectNothing(t, rc) + + // Now close connection. + c.Close() + expectNothing(t, c) + + buf := routeExpect(runsubRe) + matches := runsubRe.FindAllSubmatch(buf, -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 unsub response when closing client connection, got %d", len(matches)) + } +} + +func TestNewRouteClientClosedWithQueueSubscriptions(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:44" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + info := checkInfoMsg(t, rc) + info.ID = routeID + b, err := json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + routeSend(fmt.Sprintf("INFO %s\r\n", b)) + + routeSend("PING\r\n") + routeExpect(pongRe) + + for i := 0; i < 100; i++ { + send(fmt.Sprintf("SUB foo bar %d\r\n", i)) + } + send("PING\r\n") + expect(pongRe) + + // Queue subscribers will send all updates. + for numRSubs := 0; numRSubs != 100; { + buf := routeExpect(rsubRe) + numRSubs += len(rsubRe.FindAllSubmatch(buf, -1)) + } + + // Now close connection. + c.Close() + expectNothing(t, c) + + // We should only get one unsub for the queue subscription. + matches := runsubRe.FindAllSubmatch(routeExpect(runsubRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 unsub response when closing client connection, got %d", len(matches)) + } +} + +func TestNewRouteRUnsubAccountSpecific(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + // Allow new accounts to be created on the fly. + opts.AllowNewAccounts = true + + // Create a routeConn + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:77" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + info := checkInfoMsg(t, rc) + info.ID = routeID + b, err := json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + routeSend(fmt.Sprintf("INFO %s\r\n", b)) + + // Now create 500 subs on same subject but all different accounts. + for i := 0; i < 500; i++ { + account := fmt.Sprintf("$foo.account.%d", i) + routeSend(fmt.Sprintf("RS+ %s foo\r\n", account)) + } + routeSend("PING\r\n") + routeExpect(pongRe) + + routeSend("RS- $foo.account.22 foo\r\nPING\r\n") + routeExpect(pongRe) + + // Do not expect a message on that account. + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConnWithAccount(t, c, "$foo.account.22") + send("PUB foo 2\r\nok\r\nPING\r\n") + expect(pongRe) + c.Close() + + // But make sure we still receive on others + c = createClientConn(t, opts.Host, opts.Port) + defer c.Close() + send, expect = setupConnWithAccount(t, c, "$foo.account.33") + send("PUB foo 2\r\nok\r\nPING\r\n") + expect(pongRe) + + matches := rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkRmsg(t, matches[0], "$foo.account.33", "foo", "", "2", "ok") +} + +func TestNewRouteRSubCleanupOnDisconnect(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + // Allow new accounts to be created on the fly. + opts.AllowNewAccounts = true + + // Create a routeConn + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:77" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + info := checkInfoMsg(t, rc) + info.ID = routeID + b, err := json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + routeSend(fmt.Sprintf("INFO %s\r\n", b)) + + // Now create 100 subs on 3 different accounts. + for i := 0; i < 100; i++ { + subject := fmt.Sprintf("foo.%d", i) + routeSend(fmt.Sprintf("RS+ $foo %s\r\n", subject)) + routeSend(fmt.Sprintf("RS+ $bar %s\r\n", subject)) + routeSend(fmt.Sprintf("RS+ $baz %s bar %d\r\n", subject, i+1)) + } + routeSend("PING\r\n") + routeExpect(pongRe) + + rc.Close() + + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if ns := s.NumSubscriptions(); ns != 0 { + return fmt.Errorf("Number of subscriptions is %d", ns) + } + return nil + }) +} + +func TestNewRouteSendSubsAndMsgs(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:44" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + info := checkInfoMsg(t, rc) + info.ID = routeID + b, err := json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + routeSend(fmt.Sprintf("INFO %s\r\nPING\r\n", b)) + routeExpect(pongRe) + + // Now let's send in interest from the new protocol. + // Normal Subscription + routeSend("RS+ $G foo\r\n") + // Make sure things were processed. + routeSend("PING\r\n") + routeExpect(pongRe) + + // Now create a client and send a message, make sure we receive it + // over the route connection. + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + send("PUB foo 2\r\nok\r\nPING\r\n") + expect(pongRe) + + buf := routeExpect(rmsgRe) + matches := rmsgRe.FindAllSubmatch(buf, -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkRmsg(t, matches[0], "$G", "foo", "", "2", "ok") + + // Queue Subscription + routeSend("RS+ $G foo bar 1\r\n") + // Make sure things were processed. + routeSend("PING\r\n") + routeExpect(pongRe) + + send("PUB foo reply 2\r\nok\r\nPING\r\n") + expect(pongRe) + + matches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkRmsg(t, matches[0], "$G", "foo", "+ reply bar", "2", "ok") + + // Another Queue Subscription + routeSend("RS+ $G foo baz 1\r\n") + // Make sure things were processed. + routeSend("PING\r\n") + routeExpect(pongRe) + + send("PUB foo reply 2\r\nok\r\nPING\r\n") + expect(pongRe) + + matches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkRmsg(t, matches[0], "$G", "foo", "+ reply bar baz", "2", "ok") + + // Matching wildcard + routeSend("RS+ $G *\r\n") + // Make sure things were processed. + routeSend("PING\r\n") + routeExpect(pongRe) + + send("PUB foo reply 2\r\nok\r\nPING\r\n") + expect(pongRe) + + matches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkRmsg(t, matches[0], "$G", "foo", "+ reply bar baz", "2", "ok") + + // No reply + send("PUB foo 2\r\nok\r\nPING\r\n") + expect(pongRe) + + matches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkRmsg(t, matches[0], "$G", "foo", "| bar baz", "2", "ok") + + // Now unsubscribe from the queue group. + routeSend("RS- $G foo baz\r\n") + routeSend("RS- $G foo bar\r\n") + + routeSend("PING\r\n") + routeExpect(pongRe) + + // Now send and make sure they are removed. We should still get the message. + send("PUB foo 2\r\nok\r\nPING\r\n") + expect(pongRe) + + matches = rmsgRe.FindAllSubmatch(routeExpect(rmsgRe), -1) + if len(matches) != 1 { + t.Fatalf("Expected only 1 msg, got %d", len(matches)) + } + checkRmsg(t, matches[0], "$G", "foo", "", "2", "ok") + + routeSend("RS- $G foo\r\n") + routeSend("RS- $G *\r\n") + + routeSend("PING\r\n") + routeExpect(pongRe) + + // Now we should not receive messages anymore. + send("PUB foo 2\r\nok\r\nPING\r\n") + expect(pongRe) + + expectNothing(t, rc) +} + +func TestNewRouteProcessRoutedMsgs(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + rc := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) + defer rc.Close() + + routeID := "RTEST_NEW:55" + routeSend, routeExpect := setupRouteEx(t, rc, opts, routeID) + + info := checkInfoMsg(t, rc) + info.ID = routeID + b, err := json.Marshal(info) + if err != nil { + t.Fatalf("Could not marshal test route info: %v", err) + } + routeSend(fmt.Sprintf("INFO %s\r\nPING\r\n", b)) + routeExpect(pongRe) + + // Create a client + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + + // Normal sub to start + send("SUB foo 1\r\nPING\r\n") + expect(pongRe) + routeExpect(rsubRe) + + expectMsgs := expectMsgsCommand(t, expect) + + // Now send in a RMSG to the route and make sure its delivered to the client. + routeSend("RMSG $G foo 2\r\nok\r\nPING\r\n") + routeExpect(pongRe) + matches := expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "", "2", "ok") + + // Now send in a RMSG to the route witha reply and make sure its delivered to the client. + routeSend("RMSG $G foo reply 2\r\nok\r\nPING\r\n") + routeExpect(pongRe) + + matches = expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "reply", "2", "ok") + + // Now add in a queue subscriber for the client. + send("SUB foo bar 11\r\nPING\r\n") + expect(pongRe) + routeExpect(rsubRe) + + // Now add in another queue subscriber for the client. + send("SUB foo baz 22\r\nPING\r\n") + expect(pongRe) + routeExpect(rsubRe) + + // If we send from a route with no queues. Should only get one message. + routeSend("RMSG $G foo 2\r\nok\r\nPING\r\n") + routeExpect(pongRe) + matches = expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "", "2", "ok") + + // Now send to a specific queue group. We should get multiple messages now. + routeSend("RMSG $G foo | bar 2\r\nok\r\nPING\r\n") + routeExpect(pongRe) + matches = expectMsgs(2) + checkMsg(t, matches[0], "foo", "1", "", "2", "ok") + + // Now send to both queue groups. We should get all messages now. + routeSend("RMSG $G foo | bar baz 2\r\nok\r\nPING\r\n") + routeExpect(pongRe) + matches = expectMsgs(3) + checkMsg(t, matches[0], "foo", "1", "", "2", "ok") + + // Make sure we do the right thing with reply. + routeSend("RMSG $G foo + reply bar baz 2\r\nok\r\nPING\r\n") + routeExpect(pongRe) + matches = expectMsgs(3) + checkMsg(t, matches[0], "foo", "1", "reply", "2", "ok") +} + +func TestNewRouteQueueSubsDistribution(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendA, expectA := setupConn(t, clientA) + sendB, expectB := setupConn(t, clientB) + + // Create 100 subscribers on each server. + for i := 0; i < 100; i++ { + sproto := fmt.Sprintf("SUB foo bar %d\r\n", i) + sendA(sproto) + sendB(sproto) + } + sendA("PING\r\n") + expectA(pongRe) + + sender := createClientConn(t, optsA.Host, optsA.Port) + defer sender.Close() + send, expect := setupConn(t, sender) + + // Send 100 messages from Sender + for i := 0; i < 100; i++ { + send("PUB foo 2\r\nok\r\n") + } + send("PING\r\n") + expect(pongRe) + + numAReceived := len(msgRe.FindAllSubmatch(expectA(msgRe), -1)) + numBReceived := len(msgRe.FindAllSubmatch(expectB(msgRe), -1)) + + // We may not be able to properly time all messages being ready. + for numAReceived+numBReceived != 100 { + if buf := peek(clientB); buf != nil { + numBReceived += len(msgRe.FindAllSubmatch(buf, -1)) + } + if buf := peek(clientA); buf != nil { + numAReceived += len(msgRe.FindAllSubmatch(buf, -1)) + } + } + // These should be close to 50/50 + if numAReceived < 30 || numBReceived < 30 { + t.Fatalf("Expected numbers to be close to 50/50, got %d/%d", numAReceived, numBReceived) + } +} + +// Since we trade interest in accounts now, we have a potential issue with a new client +// connecting via a brand new account, publishing and properly doing a flush, then exiting. +// If existing subscribers were present but on a remote server they may not get the message. +func TestNewRouteSinglePublishOnNewAccount(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + //srvA.SetLogger(logger.NewTestLogger("[srvA] - ", false), true, true) + //srvB.SetLogger(logger.NewTestLogger("[srvB] - ", false), true, true) + + // Allow new accounts to be created on the fly. + optsA.AllowNewAccounts = true + optsB.AllowNewAccounts = true + + // Create and establish a listener on foo for $TEST22 account. + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConnWithAccount(t, clientA, "$TEST22") + sendA("SUB foo 1\r\nPING\r\n") + expectA(pongRe) + + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + // Send a message, flush to make sure server processed and close connection. + sendB, expectB := setupConnWithAccount(t, clientB, "$TEST22") + sendB("PUB foo 2\r\nok\r\nPING\r\n") + expectB(pongRe) + clientB.Close() + + expectMsgs := expectMsgsCommand(t, expectA) + matches := expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "", "2", "ok") +} + +// Same as above but make sure it works for queue subscribers as well. +func TestNewRouteSinglePublishToQueueSubscriberOnNewAccount(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + // Allow new accounts to be created on the fly. + optsA.AllowNewAccounts = true + optsB.AllowNewAccounts = true + + // Create and establish a listener on foo for $TEST22 account. + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConnWithAccount(t, clientA, "$TEST22") + sendA("SUB foo bar 1\r\nPING\r\n") + expectA(pongRe) + + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + // Send a message, flush to make sure server processed and close connection. + sendB, expectB := setupConnWithAccount(t, clientB, "$TEST22") + sendB("PUB foo bar 2\r\nok\r\nPING\r\n") + expectB(pongRe) + defer clientB.Close() + + expectMsgs := expectMsgsCommand(t, expectA) + matches := expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "bar", "2", "ok") + + sendB("PUB foo bar 2\r\nok\r\nPING\r\n") + expectB(pongRe) + matches = expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "bar", "2", "ok") +} + +// Same as above but make sure it works for queue subscribers over multiple routes as well. +func TestNewRouteSinglePublishToMultipleQueueSubscriberOnNewAccount(t *testing.T) { + srvA, srvB, srvC, optsA, optsB, optsC := runThreeServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + defer srvC.Shutdown() + + // Allow new accounts to be created on the fly. + optsA.AllowNewAccounts = true + optsB.AllowNewAccounts = true + optsC.AllowNewAccounts = true + + // Create and establish a listener on foo/bar for $TEST22 account. Do this on ClientA and ClientC. + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConnWithAccount(t, clientA, "$TEST22") + sendA("SUB foo bar 11\r\nPING\r\n") + expectA(pongRe) + + clientC := createClientConn(t, optsC.Host, optsC.Port) + defer clientC.Close() + + sendC, expectC := setupConnWithAccount(t, clientC, "$TEST22") + sendC("SUB foo bar 33\r\nPING\r\n") + expectC(pongRe) + + sendA("PING\r\n") + expectA(pongRe) + sendC("PING\r\n") + expectC(pongRe) + + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + time.Sleep(100 * time.Millisecond) + + // Send a message, flush to make sure server processed and close connection. + sendB, expectB := setupConnWithAccount(t, clientB, "$TEST22") + sendB("PUB foo 2\r\nok\r\nPING\r\n") + expectB(pongRe) + defer clientB.Close() + + // This should trigger either clientA or clientC, but not both.. + bufA := peek(clientA) + bufC := peek(clientC) + + if bufA != nil && bufC != nil { + t.Fatalf("Expected one or the other, but got something on both") + } + numReceived := len(msgRe.FindAllSubmatch(bufA, -1)) + numReceived += len(msgRe.FindAllSubmatch(bufC, -1)) + if numReceived != 1 { + t.Fatalf("Expected only 1 msg, got %d", numReceived) + } + + // Now make sure that we are distributing correctly between A and C + // Send 100 messages from Sender + for i := 0; i < 100; i++ { + sendB("PUB foo 2\r\nok\r\n") + } + sendB("PING\r\n") + expectB(pongRe) + + numAReceived := len(msgRe.FindAllSubmatch(expectA(msgRe), -1)) + numCReceived := len(msgRe.FindAllSubmatch(expectC(msgRe), -1)) + + // We may not be able to properly time all messages being ready. + + for numAReceived+numCReceived != 100 { + if buf := peek(clientC); buf != nil { + numCReceived += len(msgRe.FindAllSubmatch(buf, -1)) + } + if buf := peek(clientA); buf != nil { + numAReceived += len(msgRe.FindAllSubmatch(buf, -1)) + } + } + + // These should be close to 50/50 + if numAReceived < 30 || numCReceived < 30 { + t.Fatalf("Expected numbers to be close to 50/50, got %d/%d", numAReceived, numCReceived) + } +} + +func registerAccounts(t *testing.T, s *server.Server) (*server.Account, *server.Account) { + // Now create two accounts. + f, err := s.RegisterAccount("$foo") + if err != nil { + t.Fatalf("Error creating account 'foo': %v", err) + } + b, err := s.RegisterAccount("$bar") + if err != nil { + t.Fatalf("Error creating account 'bar': %v", err) + } + return f, b +} + +func addStreamExport(subject string, authorized []*server.Account, targets ...*server.Account) { + for _, acc := range targets { + acc.AddStreamExport(subject, authorized) + } +} + +func addServiceExport(subject string, authorized []*server.Account, targets ...*server.Account) { + for _, acc := range targets { + acc.AddServiceExport(subject, authorized) + } +} + +var isPublic = []*server.Account(nil) + +func TestNewRouteStreamImport(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + // Do Accounts for the servers. + fooA, _ := registerAccounts(t, srvA) + fooB, barB := registerAccounts(t, srvB) + + // Add export to both. + addStreamExport("foo", isPublic, fooA, fooB) + // Add import abilities to server B's bar account from foo. + barB.AddStreamImport(fooB, "foo", "") + + // clientA will be connected to srvA and be the stream producer. + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConnWithAccount(t, clientA, "$foo") + + // Now setup client B on srvB who will do a sub from account $bar + // that should map account $foo's foo subject. + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendB, expectB := setupConnWithAccount(t, clientB, "$bar") + sendB("SUB foo 1\r\nPING\r\n") + expectB(pongRe) + + // Send on clientA + sendA("PING\r\n") + expectA(pongRe) + + sendA("PUB foo 2\r\nok\r\nPING\r\n") + expectA(pongRe) + + expectMsgs := expectMsgsCommand(t, expectB) + matches := expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "", "2", "ok") + + // Send Again on clientA + sendA("PUB foo 2\r\nok\r\nPING\r\n") + expectA(pongRe) + + matches = expectMsgs(1) + checkMsg(t, matches[0], "foo", "1", "", "2", "ok") + + sendB("UNSUB 1\r\nPING\r\n") + expectB(pongRe) + + sendA("PUB foo 2\r\nok\r\nPING\r\n") + expectA(pongRe) + expectNothing(t, clientA) +} + +func TestNewRouteReservedReply(t *testing.T) { + s, opts := runNewRouteServer(t) + defer s.Shutdown() + + c := createClientConn(t, opts.Host, opts.Port) + defer c.Close() + + send, expect := setupConn(t, c) + + // Test that clients can't send to reserved service import replies. + send("PUB foo _R_.foo 2\r\nok\r\nPING\r\n") + expect(errRe) +} + +func TestNewRouteServiceImport(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + //srvA.SetLogger(logger.NewTestLogger("[srvA] - ", false), true, true) + //srvB.SetLogger(logger.NewTestLogger("[srvB] - ", false), true, true) + + // Do Accounts for the servers. + fooA, _ := registerAccounts(t, srvA) + fooB, barB := registerAccounts(t, srvB) + + // Add in the service export for the requests. Make it public. + if err := fooA.AddServiceExport("test.request", nil); err != nil { + t.Fatalf("Error adding account service export to client foo: %v", err) + } + + // Add export to both. + addServiceExport("test.request", isPublic, fooA, fooB) + + // Add import abilities to server B's bar account from foo. + if err := barB.AddServiceImport(fooB, "foo.request", "test.request"); err != nil { + t.Fatalf("Error adding service import: %v", err) + } + + // clientA will be connected to srvA and be the service endpoint and responder. + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConnWithAccount(t, clientA, "$foo") + sendA("SUB test.request 1\r\nPING\r\n") + expectA(pongRe) + + // Now setup client B on srvB who will do a sub from account $bar + // that should map account $foo's foo subject. + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendB, expectB := setupConnWithAccount(t, clientB, "$bar") + sendB("SUB reply 1\r\nPING\r\n") + expectB(pongRe) + + // Send the request from clientB on foo.request, + sendB("PUB foo.request reply 2\r\nhi\r\nPING\r\n") + expectB(pongRe) + + expectMsgsA := expectMsgsCommand(t, expectA) + expectMsgsB := expectMsgsCommand(t, expectB) + + // Expect the request on A + matches := expectMsgsA(1) + reply := string(matches[0][replyIndex]) + checkMsg(t, matches[0], "test.request", "1", reply, "2", "hi") + if reply == "reply" { + t.Fatalf("Expected randomized reply, but got original") + } + + sendA(fmt.Sprintf("PUB %s 2\r\nok\r\nPING\r\n", reply)) + expectA(pongRe) + + matches = expectMsgsB(1) + checkMsg(t, matches[0], "reply", "1", "", "2", "ok") + + if ts := fooA.TotalSubs(); ts != 1 { + t.Fatalf("Expected one sub to be left on fooA, but got %d", ts) + } +} + +func TestNewRouteServiceImportDanglingRemoteSubs(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + //srvA.SetLogger(logger.NewTestLogger("[srvA] - ", false), true, true) + //srvB.SetLogger(logger.NewTestLogger("[srvB] - ", false), true, true) + + // Do Accounts for the servers. + fooA, _ := registerAccounts(t, srvA) + fooB, barB := registerAccounts(t, srvB) + + fooA.SetAutoExpireTTL(10 * time.Millisecond) + + // Add in the service export for the requests. Make it public. + if err := fooA.AddServiceExport("test.request", nil); err != nil { + t.Fatalf("Error adding account service export to client foo: %v", err) + } + + // Add export to both. + addServiceExport("test.request", isPublic, fooA, fooB) + + // Add import abilities to server B's bar account from foo. + if err := barB.AddServiceImport(fooB, "foo.request", "test.request"); err != nil { + t.Fatalf("Error adding service import: %v", err) + } + + // clientA will be connected to srvA and be the service endpoint, but will not send responses. + clientA := createClientConn(t, optsA.Host, optsA.Port) + defer clientA.Close() + + sendA, expectA := setupConnWithAccount(t, clientA, "$foo") + // Express interest. + sendA("SUB test.request 1\r\nPING\r\n") + expectA(pongRe) + + // Now setup client B on srvB who will do a sub from account $bar + // that should map account $foo's foo subject. + clientB := createClientConn(t, optsB.Host, optsB.Port) + defer clientB.Close() + + sendB, expectB := setupConnWithAccount(t, clientB, "$bar") + sendB("SUB reply 1\r\nPING\r\n") + expectB(pongRe) + + // Send 100 requests from clientB on foo.request, + for i := 0; i < 100; i++ { + sendB("PUB foo.request reply 2\r\nhi\r\n") + } + sendB("PING\r\n") + expectB(pongRe) + + numRequests := 0 + // Expect the request on A + checkFor(t, time.Second, 10*time.Millisecond, func() error { + buf := expectA(msgRe) + matches := msgRe.FindAllSubmatch(buf, -1) + numRequests += len(matches) + if numRequests != 100 { + return fmt.Errorf("Number of requests is %d", numRequests) + } + return nil + }) + + expectNothing(t, clientB) + + // These reply subjects will be dangling off of $foo account on serverA. + // Remove our service endpoint and wait for the dangling replies to go to zero. + sendA("UNSUB 1\r\nPING\r\n") + expectA(pongRe) + + checkFor(t, time.Second, 10*time.Millisecond, func() error { + if ts := fooA.TotalSubs(); ts != 0 { + return fmt.Errorf("Number of subs is %d, should be zero", ts) + } + return nil + }) +} + +func TestNewRouteNoQueueSubscribersBounce(t *testing.T) { + srvA, srvB, optsA, optsB := runServers(t) + defer srvA.Shutdown() + defer srvB.Shutdown() + + urlA := fmt.Sprintf("nats://%s:%d/", optsA.Host, optsA.Port) + urlB := fmt.Sprintf("nats://%s:%d/", optsB.Host, optsB.Port) + + ncA, err := nats.Connect(urlA) + if err != nil { + t.Fatalf("Failed to create connection for ncA: %v\n", err) + } + defer ncA.Close() + + ncB, err := nats.Connect(urlB) + if err != nil { + t.Fatalf("Failed to create connection for ncB: %v\n", err) + } + defer ncB.Close() + + response := []byte("I will help you") + + // Create a lot of queue subscribers on A, and have one on B. + ncB.QueueSubscribe("foo.request", "workers", func(m *nats.Msg) { + ncB.Publish(m.Reply, response) + }) + + for i := 0; i < 100; i++ { + ncA.QueueSubscribe("foo.request", "workers", func(m *nats.Msg) { + ncA.Publish(m.Reply, response) + }) + } + ncB.Flush() + ncA.Flush() + + // Send all requests from B + numAnswers := 0 + for i := 0; i < 500; i++ { + if _, err := ncB.Request("foo.request", []byte("Help Me"), time.Second); err != nil { + t.Fatalf("Received an error on Request test [%d]: %s", i, err) + } + numAnswers++ + // After we have sent 20 go into drain mode for the ncA client. + if i == 20 { + ncA.Close() + } + } + + if numAnswers != 500 { + t.Fatalf("Expect to get all 500 responses, got %d", numAnswers) + } +} diff --git a/test/routes_test.go b/test/routes_test.go index ab49544a..5e1f5469 100644 --- a/test/routes_test.go +++ b/test/routes_test.go @@ -21,7 +21,6 @@ import ( "os" "runtime" "strconv" - "strings" "sync" "testing" "time" @@ -148,25 +147,14 @@ func TestSendRouteSubAndUnsub(t *testing.T) { // Send SUB via client connection send("SUB foo 22\r\n") - // Make sure the SUB is broadcast via the route - buf := expectResult(t, rc, subRe) - matches := subRe.FindAllSubmatch(buf, -1) - rsid := string(matches[0][5]) - if !strings.HasPrefix(rsid, "RSID:") { - t.Fatalf("Got wrong RSID: %s\n", rsid) - } + // Make sure the RS+ is broadcast via the route + expectResult(t, rc, rsubRe) // Send UNSUB via client connection send("UNSUB 22\r\n") - // Make sure the SUB is broadcast via the route - buf = expectResult(t, rc, unsubRe) - matches = unsubRe.FindAllSubmatch(buf, -1) - rsid2 := string(matches[0][1]) - - if rsid2 != rsid { - t.Fatalf("Expected rsid's to match. %q vs %q\n", rsid, rsid2) - } + // Make sure the RS- is broadcast via the route + expectResult(t, rc, runsubRe) // Explicitly shutdown the server, otherwise this test would // cause following test to fail. @@ -217,18 +205,16 @@ func TestRouteForwardsMsgFromClients(t *testing.T) { routeExpect(infoRe) } - // Send SUB via route connection - routeSend("SUB foo RSID:2:22\r\n") - routeSend("PING\r\n") + // Send SUB via route connection, RS+ + routeSend("RS+ $G foo\r\nPING\r\n") routeExpect(pongRe) // Send PUB via client connection - clientSend("PUB foo 2\r\nok\r\n") - clientSend("PING\r\n") + clientSend("PUB foo 2\r\nok\r\nPING\r\n") clientExpect(pongRe) matches := expectMsgs(1) - checkMsg(t, matches[0], "foo", "RSID:2:22", "", "2", "ok") + checkRmsg(t, matches[0], "$G", "foo", "", "2", "ok") } func TestRouteForwardsMsgToClients(t *testing.T) { @@ -247,13 +233,12 @@ func TestRouteForwardsMsgToClients(t *testing.T) { routeSend, _ := setupRoute(t, route, opts) // Subscribe to foo - clientSend("SUB foo 1\r\n") + clientSend("SUB foo 1\r\nPING\r\n") // Use ping roundtrip to make sure its processed. - clientSend("PING\r\n") clientExpect(pongRe) - // Send MSG proto via route connection - routeSend("MSG foo 1 2\r\nok\r\n") + // Send RMSG proto via route connection + routeSend("RMSG $G foo 2\r\nok\r\n") matches := expectMsgs(1) checkMsg(t, matches[0], "foo", "1", "", "2", "ok") @@ -270,10 +255,10 @@ func TestRouteOneHopSemantics(t *testing.T) { routeSend, _ := setupRoute(t, route, opts) // Express interest on this route for foo. - routeSend("SUB foo RSID:2:2\r\n") + routeSend("RS+ $G foo\r\n") // Send MSG proto via route connection - routeSend("MSG foo 1 2\r\nok\r\n") + routeSend("RMSG foo 2\r\nok\r\n") // Make sure it does not come back! expectNothing(t, route) @@ -296,14 +281,13 @@ func TestRouteOnlySendOnce(t *testing.T) { expectMsgs := expectMsgsCommand(t, routeExpect) // Express multiple interest on this route for foo. - routeSend("SUB foo RSID:2:1\r\n") - routeSend("SUB foo RSID:2:2\r\n") + routeSend("RS+ $G foo\r\n") + routeSend("RS+ $G foo\r\n") routeSend("PING\r\n") routeExpect(pongRe) // Send PUB via client connection - clientSend("PUB foo 2\r\nok\r\n") - clientSend("PING\r\n") + clientSend("PUB foo 2\r\nok\r\nPING\r\n") clientExpect(pongRe) expectMsgs(1) @@ -327,13 +311,11 @@ func TestRouteQueueSemantics(t *testing.T) { expectAuthRequired(t, route) routeSend, routeExpect := setupRouteEx(t, route, opts, "ROUTER:xyz") routeSend("INFO {\"server_id\":\"ROUTER:xyz\"}\r\n") - expectMsgs := expectMsgsCommand(t, routeExpect) + expectMsgs := expectRmsgsCommand(t, routeExpect) // Express multiple interest on this route for foo, queue group bar. - qrsid1 := "QRSID:1:1" - routeSend(fmt.Sprintf("SUB foo bar %s\r\n", qrsid1)) - qrsid2 := "QRSID:1:2" - routeSend(fmt.Sprintf("SUB foo bar %s\r\n", qrsid2)) + routeSend("RS+ $G foo bar 1\r\n") + routeSend("RS+ $G foo bar 2\r\n") // Use ping roundtrip to make sure its processed. routeSend("PING\r\n") @@ -347,75 +329,48 @@ func TestRouteQueueSemantics(t *testing.T) { // Only 1 matches := expectMsgs(1) - checkMsg(t, matches[0], "foo", "", "", "2", "ok") + checkRmsg(t, matches[0], "$G", "foo", "| bar", "2", "ok") // Add normal Interest as well to route interest. - routeSend("SUB foo RSID:1:4\r\n") + routeSend("RS+ $G foo\r\n") // Use ping roundtrip to make sure its processed. routeSend("PING\r\n") routeExpect(pongRe) // Send PUB via client connection - clientSend("PUB foo 2\r\nok\r\n") + clientSend("PUB foo 2\r\nok\r\nPING\r\n") // Use ping roundtrip to make sure its processed. - clientSend("PING\r\n") clientExpect(pongRe) - // Should be 2 now, 1 for all normal, and one for specific queue subscriber. - matches = expectMsgs(2) - - // Expect first to be the normal subscriber, next will be the queue one. - if string(matches[0][sidIndex]) != "RSID:1:4" && - string(matches[1][sidIndex]) != "RSID:1:4" { - t.Fatalf("Did not received routed sid\n") - } - checkMsg(t, matches[0], "foo", "", "", "2", "ok") - checkMsg(t, matches[1], "foo", "", "", "2", "ok") - - // Check the rsid to verify it is one of the queue group subscribers. - var rsid string - if matches[0][sidIndex][0] == 'Q' { - rsid = string(matches[0][sidIndex]) - } else { - rsid = string(matches[1][sidIndex]) - } - if rsid != qrsid1 && rsid != qrsid2 { - t.Fatalf("Expected a queue group rsid, got %s\n", rsid) - } + // Should be 1 now for everything. Always receive 1 message. + matches = expectMsgs(1) + checkRmsg(t, matches[0], "$G", "foo", "| bar", "2", "ok") // Now create a queue subscription for the client as well as a normal one. clientSend("SUB foo 1\r\n") // Use ping roundtrip to make sure its processed. clientSend("PING\r\n") clientExpect(pongRe) - routeExpect(subRe) + routeExpect(rsubRe) - clientSend("SUB foo bar 2\r\n") + clientSend("SUB foo bar 2\r\nPING\r\n") // Use ping roundtrip to make sure its processed. - clientSend("PING\r\n") clientExpect(pongRe) - routeExpect(subRe) + routeExpect(rsubRe) // Deliver a MSG from the route itself, make sure the client receives both. - routeSend("MSG foo RSID:1:1 2\r\nok\r\n") - // Queue group one. - routeSend("MSG foo QRSID:1:2 2\r\nok\r\n") - // Invlaid queue sid. - routeSend("MSG foo QRSID 2\r\nok\r\n") // cid and sid missing - routeSend("MSG foo QRSID:1 2\r\nok\r\n") // cid not terminated with ':' - routeSend("MSG foo QRSID:1: 2\r\nok\r\n") // cid==1 but sid missing. It needs to be at least one character. + routeSend("RMSG $G foo | bar 2\r\nok\r\n") // Use ping roundtrip to make sure its processed. routeSend("PING\r\n") routeExpect(pongRe) - // Should be 2 now, 1 for all normal, and one for specific queue subscriber. + // Should get 2 msgs. matches = clientExpectMsgs(2) // Expect first to be the normal subscriber, next will be the queue one. - checkMsg(t, matches[0], "foo", "1", "", "2", "ok") - checkMsg(t, matches[1], "foo", "2", "", "2", "ok") + checkMsg(t, matches[0], "foo", "", "", "2", "ok") } func TestSolicitRouteReconnect(t *testing.T) { @@ -442,22 +397,24 @@ func TestMultipleRoutesSameId(t *testing.T) { defer route1.Close() expectAuthRequired(t, route1) - route1Send, _ := setupRouteEx(t, route1, opts, "ROUTE:2222") + route1Send, route1Expect := setupRouteEx(t, route1, opts, "ROUTE:2222") route2 := createRouteConn(t, opts.Cluster.Host, opts.Cluster.Port) defer route2.Close() expectAuthRequired(t, route2) - route2Send, _ := setupRouteEx(t, route2, opts, "ROUTE:2222") + route2Send, route2Expect := setupRouteEx(t, route2, opts, "ROUTE:2222") // Send SUB via route connections - sub := "SUB foo RSID:2:22\r\n" + sub := "RS+ $G foo\r\nPING\r\n" route1Send(sub) route2Send(sub) + route1Expect(pongRe) + route2Expect(pongRe) - // Make sure we do not get anything on a MSG send to a router. - // Send MSG proto via route connection - route1Send("MSG foo 1 2\r\nok\r\n") + // Make sure we do not get anything on a RMSG send to a router. + // Send RMSG proto via route connection + route1Send("RMSG $G foo 2\r\nok\r\n") expectNothing(t, route1) expectNothing(t, route2) @@ -468,8 +425,7 @@ func TestMultipleRoutesSameId(t *testing.T) { defer client.Close() // Send PUB via client connection - clientSend("PUB foo 2\r\nok\r\n") - clientSend("PING\r\n") + clientSend("PUB foo 2\r\nok\r\nPING\r\n") clientExpect(pongRe) // We should only receive on one route, not both. @@ -486,11 +442,11 @@ func TestMultipleRoutesSameId(t *testing.T) { } } - matches := msgRe.FindAllSubmatch(buf, -1) + matches := rmsgRe.FindAllSubmatch(buf, -1) if len(matches) != 1 { t.Fatalf("Expected 1 msg, got %d\n", len(matches)) } - checkMsg(t, matches[0], "foo", "", "", "2", "ok") + checkRmsg(t, matches[0], "$G", "foo", "", "2", "ok") } func TestRouteResendsLocalSubsOnReconnect(t *testing.T) { @@ -530,7 +486,7 @@ func TestRouteResendsLocalSubsOnReconnect(t *testing.T) { // Trigger the send of local subs. routeSend(infoJSON) - routeExpect(subRe) + routeExpect(rsubRe) // Close and then re-open route.Close() @@ -543,7 +499,7 @@ func TestRouteResendsLocalSubsOnReconnect(t *testing.T) { routeExpect(infoRe) routeSend(infoJSON) - routeExpect(subRe) + routeExpect(rsubRe) } type ignoreLogger struct{} @@ -990,8 +946,8 @@ func TestRouteBasicPermissions(t *testing.T) { if err := checkExpectedSubs(5, srvA); err != nil { t.Fatal(err.Error()) } - // B should have 4 - if err := checkExpectedSubs(4, srvB); err != nil { + // B should have 3 since we coalesce te two for 'foo' + if err := checkExpectedSubs(3, srvB); err != nil { t.Fatal(err.Error()) } // Send a message from B and check that it is received. @@ -1011,7 +967,7 @@ func TestRouteBasicPermissions(t *testing.T) { ncb.Close() srvB.Shutdown() - // Since B had 2 local subs, A should go from 5 to 3 + // Since B had 2 local subs, A should still only go from 4 to 3 if err := checkExpectedSubs(3, srvA); err != nil { t.Fatal(err.Error()) } @@ -1020,8 +976,8 @@ func TestRouteBasicPermissions(t *testing.T) { srvB, optsB = RunServerWithConfig("./configs/srv_b.conf") defer srvB.Shutdown() // Check that subs from A that can be sent to B are sent. - // That would be 2 (the 2 subscriptions on foo). - if err := checkExpectedSubs(2, srvB); err != nil { + // That would be 2 (the 2 subscriptions on foo) as one. + if err := checkExpectedSubs(1, srvB); err != nil { t.Fatal(err.Error()) } diff --git a/test/test.go b/test/test.go index adaeaa13..932a018d 100644 --- a/test/test.go +++ b/test/test.go @@ -69,13 +69,12 @@ func RunServer(opts *server.Options) *server.Server { } // LoadConfig loads a configuration from a filename -func LoadConfig(configFile string) (opts *server.Options) { +func LoadConfig(configFile string) *server.Options { opts, err := server.ProcessConfigFile(configFile) if err != nil { panic(fmt.Sprintf("Error processing configuration file: %v", err)) } - opts.NoSigs, opts.NoLog = true, true - return + return opts } // RunServerWithConfig starts a new Go routine based server with a configuration file. @@ -215,6 +214,13 @@ func setupConnWithProto(t tLogger, c net.Conn, proto int) (sendFun, expectFun) { return sendCommand(t, c), expectCommand(t, c) } +func setupConnWithAccount(t tLogger, c net.Conn, account string) (sendFun, expectFun) { + checkInfoMsg(t, c) + cs := fmt.Sprintf("CONNECT {\"verbose\":%v,\"pedantic\":%v,\"tls_required\":%v,\"account\":%q}\r\n", false, false, false, account) + sendProto(t, c, cs) + return sendCommand(t, c), expectCommand(t, c) +} + type sendFun func(string) type expectFun func(*regexp.Regexp) []byte @@ -250,24 +256,31 @@ var ( msgRe = regexp.MustCompile(`(?:(?:MSG\s+([^\s]+)\s+([^\s]+)\s+(([^\s]+)[^\S\r\n]+)?(\d+)\s*\r\n([^\\r\\n]*?)\r\n)+?)`) okRe = regexp.MustCompile(`\A\+OK\r\n`) errRe = regexp.MustCompile(`\A\-ERR\s+([^\r\n]+)\r\n`) - subRe = regexp.MustCompile(`SUB\s+([^\s]+)((\s+)([^\s]+))?\s+([^\s]+)\r\n`) - unsubRe = regexp.MustCompile(`UNSUB\s+([^\s]+)(\s+(\d+))?\r\n`) connectRe = regexp.MustCompile(`CONNECT\s+([^\r\n]+)\r\n`) + rsubRe = regexp.MustCompile(`RS\+\s+([^\s]+)\s+([^\s]+)\s*([^\s]+)?\s*(\d+)?\r\n`) + runsubRe = regexp.MustCompile(`RS\-\s+([^\s]+)\s+([^\s]+)\s*([^\s]+)?\r\n`) + rmsgRe = regexp.MustCompile(`(?:(?:RMSG\s+([^\s]+)\s+([^\s]+)\s+(?:([|+]\s+([\w\s]+)|[^\s]+)[^\S\r\n]+)?(\d+)\s*\r\n([^\\r\\n]*?)\r\n)+?)`) ) const ( + // Regular Messages subIndex = 1 sidIndex = 2 replyIndex = 4 lenIndex = 5 msgIndex = 6 + + // Routed Messages + accIndex = 1 + rsubIndex = 2 + replyAndQueueIndex = 3 ) // Test result from server against regexp func expectResult(t tLogger, c net.Conn, re *regexp.Regexp) []byte { expBuf := make([]byte, 32768) // Wait for commands to be processed and results queued for read - c.SetReadDeadline(time.Now().Add(5 * time.Second)) + c.SetReadDeadline(time.Now().Add(2 * time.Second)) n, err := c.Read(expBuf) c.SetReadDeadline(time.Time{}) @@ -277,11 +290,22 @@ func expectResult(t tLogger, c net.Conn, re *regexp.Regexp) []byte { buf := expBuf[:n] if !re.Match(buf) { - stackFatalf(t, "Response did not match expected: \n\tReceived:'%q'\n\tExpected:'%s'\n", buf, re) + stackFatalf(t, "Response did not match expected: \n\tReceived:'%q'\n\tExpected:'%s'", buf, re) } return buf } +func peek(c net.Conn) []byte { + expBuf := make([]byte, 32768) + c.SetReadDeadline(time.Now().Add(50 * time.Millisecond)) + n, err := c.Read(expBuf) + c.SetReadDeadline(time.Time{}) + if err != nil || n <= 0 { + return nil + } + return expBuf +} + func expectNothing(t tLogger, c net.Conn) { expBuf := make([]byte, 32) c.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) @@ -292,7 +316,7 @@ func expectNothing(t tLogger, c net.Conn) { } } -// This will check that we got what we expected. +// This will check that we got what we expected from a normal message. func checkMsg(t tLogger, m [][]byte, subject, sid, reply, len, msg string) { if string(m[subIndex]) != subject { stackFatalf(t, "Did not get correct subject: expected '%s' got '%s'\n", subject, m[subIndex]) @@ -311,6 +335,33 @@ func checkMsg(t tLogger, m [][]byte, subject, sid, reply, len, msg string) { } } +func checkRmsg(t tLogger, m [][]byte, account, subject, replyAndQueues, len, msg string) { + if string(m[accIndex]) != account { + stackFatalf(t, "Did not get correct account: expected '%s' got '%s'\n", account, m[accIndex]) + } + if string(m[rsubIndex]) != subject { + stackFatalf(t, "Did not get correct subject: expected '%s' got '%s'\n", subject, m[rsubIndex]) + } + if string(m[lenIndex]) != len { + stackFatalf(t, "Did not get correct msg length: expected '%s' got '%s'\n", len, m[lenIndex]) + } + if string(m[replyAndQueueIndex]) != replyAndQueues { + stackFatalf(t, "Did not get correct reply/queues: expected '%s' got '%s'\n", replyAndQueues, m[replyAndQueueIndex]) + } +} + +// Closure for expectMsgs +func expectRmsgsCommand(t tLogger, ef expectFun) func(int) [][][]byte { + return func(expected int) [][][]byte { + buf := ef(rmsgRe) + matches := rmsgRe.FindAllSubmatch(buf, -1) + if len(matches) != expected { + stackFatalf(t, "Did not get correct # routed msgs: %d vs %d\n", len(matches), expected) + } + return matches + } +} + // Closure for expectMsgs func expectMsgsCommand(t tLogger, ef expectFun) func(int) [][][]byte { return func(expected int) [][][]byte { diff --git a/test/user_authorization_test.go b/test/user_authorization_test.go index 53cf1f47..b334e94a 100644 --- a/test/user_authorization_test.go +++ b/test/user_authorization_test.go @@ -37,10 +37,6 @@ func TestUserAuthorizationProto(t *testing.T) { sendProto(t, c, "SUB foo 1\r\n") expectResult(t, c, okRe) - // Check that we now reserve _SYS.> though for internal, so no clients. - sendProto(t, c, "PUB _SYS.HB 2\r\nok\r\n") - expectResult(t, c, permErrRe) - // Check that _ is ok sendProto(t, c, "PUB _ 2\r\nok\r\n") expectResult(t, c, okRe)