Fix missing capture of remoteID in removeClient

This needed to be captured under the client lock. Adding a test
that captured the races (when running enough times).

server/server.go

@@ -690,10 +690,14 @@ func (s *Server) checkAuth(c *client) bool {
 
 // Remove a client or route from our internal accounting.
 func (s *Server) removeClient(c *client) {
+	var rID string
 	c.mu.Lock()
 	cid := c.cid
 	typ := c.typ
 	r := c.route
+	if r != nil {
+		rID = r.remoteID
+	}
 	c.mu.Unlock()
 
 	s.mu.Lock()
@@ -703,10 +707,10 @@ func (s *Server) removeClient(c *client) {
 	case ROUTER:
 		delete(s.routes, cid)
 		if r != nil {
-			rc, ok := s.remotes[r.remoteID]
+			rc, ok := s.remotes[rID]
 			// Only delete it if it is us..
 			if ok && c == rc {
-				delete(s.remotes, r.remoteID)
+				delete(s.remotes, rID)
 			}
 		}
 	}

test/routes_test.go

@@ -9,6 +9,7 @@ import (
 	"net"
 	"runtime"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -611,3 +612,47 @@ func TestAutoUnsubPropagation(t *testing.T) {
 
 	routeExpect(unsubnomaxRe)
 }
+
+type ignoreLogger struct {
+}
+
+func (l *ignoreLogger) Fatalf(f string, args ...interface{}) {
+}
+func (l *ignoreLogger) Errorf(f string, args ...interface{}) {
+}
+
+func TestRouteConnectOnShutdownRace(t *testing.T) {
+	s, opts := runRouteServer(t)
+	defer s.Shutdown()
+
+	l := &ignoreLogger{}
+
+	var wg sync.WaitGroup
+
+	cQuit := make(chan bool, 1)
+
+	wg.Add(1)
+
+	go func() {
+		defer wg.Done()
+		for {
+			route := createRouteConn(l, opts.ClusterHost, opts.ClusterPort)
+			if route != nil {
+				setupRouteEx(l, route, opts, "ROUTE:4222")
+				route.Close()
+			}
+			select {
+			case <-cQuit:
+				return
+			default:
+			}
+		}
+	}()
+
+	time.Sleep(5 * time.Millisecond)
+	s.Shutdown()
+
+	cQuit <- true
+
+	wg.Wait()
+}