[Added] account specific monitoring endpoint(s) (#3250)

Added http monitoring endpoint /accstatz
It responds with a list of statz for all accounts with local connections
the argument "unused=1" can be provided to get statz for all accounts
This endpoint is also exposed as nats request under:

This monitoring endpoint is exposed via the system account.
$SYS.REQ.ACCOUNT.*.STATZ
Each server will respond with connection statistics for the requested
account. The format of the data section is a list (size 1) identical to the event
$SYS.ACCOUNT.%s.SERVER.CONNS which is sent periodically as well as on
connect/disconnect. Unless requested by options, server without the account,
or server where the account has no local connections, will not respond.

A PING endpoint exists as well. The response format is identical to
$SYS.REQ.ACCOUNT.*.STATZ
(however the data section will contain more than one account, if they exist)
In addition to general filter options the request takes a list of accounts and
an argument to include accounts without local connections (disabled by default)
$SYS.REQ.ACCOUNT.PING.STATZ

Each account has a new system account import where the local subject
$SYS.REQ.ACCOUNT.PING.STATZ essentially responds as if
the importing account name was used for $SYS.REQ.ACCOUNT.*.STATZ

The only difference between requesting ACCOUNT.PING.STATZ from within
the system account and an account is that the later can only retrieve
statz for the account the client requests from.

Also exposed the monitoring /healthz via the system account under
$SYS.REQ.SERVER.*.HEALTHZ
$SYS.REQ.SERVER.PING.HEALTHZ
No dedicated options are available for these.
HEALTHZ also accept general filter options.

Signed-off-by: Matthias Hanel <mh@synadia.com>

server/monitor.go

@@ -92,10 +92,6 @@ type ConnzOptions struct {
 
 	// Filter by subject interest
 	FilterSubject string `json:"filter_subject"`
-
-	// Private indication that this request is from an account and not a system account.
-	// Used to not leak system level information to the account.
-	isAccountReq bool
 }
 
 // ConnState is for filtering states of connections. We will only have two, open and closed.
@@ -256,9 +252,6 @@ func (s *Server) Connz(opts *ConnzOptions) (*Connz, error) {
 
 	var clist map[uint64]*client
 
-	// If this is an account scoped request from a no $SYS account.
-	isAccReq := acc != _EMPTY_ && opts.isAccountReq
-
 	if acc != _EMPTY_ {
 		var err error
 		a, err = s.lookupAccount(acc)
@@ -300,7 +293,7 @@ func (s *Server) Connz(opts *ConnzOptions) (*Connz, error) {
 	}
 
 	// We may need to filter these connections.
-	if isAccReq && len(closedClients) > 0 {
+	if acc != _EMPTY_ && len(closedClients) > 0 {
 		var ccc []*closedClient
 		for _, cc := range closedClients {
 			if cc.acc == acc {
@@ -1348,6 +1341,7 @@ func (s *Server) HandleRoot(w http.ResponseWriter, r *http.Request) {
 	<a href=.%s>JetStream</a>
 	<a href=.%s>Connections</a>
 	<a href=.%s>Accounts</a>
+	<a href=.%s>Account stats</a>
 	<a href=.%s>Subscriptions</a>
 	<a href=.%s>Routes</a>
 	<a href=.%s>LeafNodes</a>
@@ -1362,6 +1356,7 @@ func (s *Server) HandleRoot(w http.ResponseWriter, r *http.Request) {
 		s.basePath(JszPath),
 		s.basePath(ConnzPath),
 		s.basePath(AccountzPath),
+		s.basePath(AccountStatzPath),
 		s.basePath(SubszPath),
 		s.basePath(RoutezPath),
 		s.basePath(LeafzPath),
@@ -2156,6 +2151,78 @@ func (s *Server) HandleLeafz(w http.ResponseWriter, r *http.Request) {
 	ResponseHandler(w, r, b)
 }
 
+// Leafz represents detailed information on Leafnodes.
+type AccountStatz struct {
+	ID       string         `json:"server_id"`
+	Now      time.Time      `json:"now"`
+	Accounts []*AccountStat `json:"account_statz"`
+}
+
+// LeafzOptions are options passed to Leafz
+type AccountStatzOptions struct {
+	Accounts      []string `json:"accounts"`
+	IncludeUnused bool     `json:"include_unused"`
+}
+
+// Leafz returns a AccountStatz structure containing summary information about accounts.
+func (s *Server) AccountStatz(opts *AccountStatzOptions) (*AccountStatz, error) {
+	stz := &AccountStatz{
+		ID:       s.ID(),
+		Now:      time.Now().UTC(),
+		Accounts: []*AccountStat{},
+	}
+	if opts == nil || len(opts.Accounts) == 0 {
+		s.accounts.Range(func(key, a interface{}) bool {
+			acc := a.(*Account)
+			acc.mu.RLock()
+			if opts.IncludeUnused || acc.numLocalConnections() != 0 {
+				stz.Accounts = append(stz.Accounts, acc.statz())
+			}
+			acc.mu.RUnlock()
+			return true
+		})
+	} else {
+		for _, a := range opts.Accounts {
+			if acc, ok := s.accounts.Load(a); ok {
+				acc := acc.(*Account)
+				acc.mu.RLock()
+				if opts.IncludeUnused || acc.numLocalConnections() != 0 {
+					stz.Accounts = append(stz.Accounts, acc.statz())
+				}
+				acc.mu.RUnlock()
+			}
+		}
+	}
+	return stz, nil
+}
+
+// HandleAccountStatz process HTTP requests for statz information of all accounts.
+func (s *Server) HandleAccountStatz(w http.ResponseWriter, r *http.Request) {
+	s.mu.Lock()
+	s.httpReqStats[AccountStatzPath]++
+	s.mu.Unlock()
+
+	unused, err := decodeBool(w, r, "unused")
+	if err != nil {
+		return
+	}
+
+	l, err := s.AccountStatz(&AccountStatzOptions{IncludeUnused: unused})
+	if err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		w.Write([]byte(err.Error()))
+		return
+	}
+	b, err := json.MarshalIndent(l, "", "  ")
+	if err != nil {
+		s.Errorf("Error marshaling response to %s request: %v", AccountStatzPath, err)
+		return
+	}
+
+	// Handle response
+	ResponseHandler(w, r, b)
+}
+
 // ResponseHandler handles responses for monitoring routes
 func ResponseHandler(w http.ResponseWriter, r *http.Request, data []byte) {
 	// Get callback from request