mirror of
https://github.com/gogrlx/nats-server.git
synced 2026-04-02 03:38:42 -07:00
5050092468
When updating usage, there is a lock inversion in that the jetStream lock was acquired while under the stream's (mset) lock, which is not correct. Also, updateUsage was locking the jsAccount lock, which again, is not really correct since jsAccount contains streams, so it should be jsAccount->stream, not the other way around. Removed the locking of jetStream to check for clustered state since js.clustered is immutable. Replaced using jsAccount lock to update usage with a dedicated lock. Originally moved all the update/limit fields in jsAccount to new structure to make sure that I would see all code that is updating or reading those fields, and also all functions so that I could make sure that I use the new lock when calling these. Once that works was done, and to reduce code changes, I put the fields back into jsAccount (although I grouped them under the new usageMu mutex field). Signed-off-by: Ivan Kozlovic <ivan@synadia.com>
2939 lines
84 KiB
Go
2939 lines
84 KiB
Go
// Copyright 2013-2022 The NATS Authors
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package server
|
|
|
|
import (
|
|
"crypto/tls"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"runtime"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/nats-io/jwt/v2"
|
|
"github.com/nats-io/nats-server/v2/server/pse"
|
|
)
|
|
|
|
// Snapshot this
|
|
var numCores int
|
|
var maxProcs int
|
|
|
|
func init() {
|
|
numCores = runtime.NumCPU()
|
|
maxProcs = runtime.GOMAXPROCS(0)
|
|
}
|
|
|
|
// Connz represents detailed information on current client connections.
|
|
type Connz struct {
|
|
ID string `json:"server_id"`
|
|
Now time.Time `json:"now"`
|
|
NumConns int `json:"num_connections"`
|
|
Total int `json:"total"`
|
|
Offset int `json:"offset"`
|
|
Limit int `json:"limit"`
|
|
Conns []*ConnInfo `json:"connections"`
|
|
}
|
|
|
|
// ConnzOptions are the options passed to Connz()
|
|
type ConnzOptions struct {
|
|
// Sort indicates how the results will be sorted. Check SortOpt for possible values.
|
|
// Only the sort by connection ID (ByCid) is ascending, all others are descending.
|
|
Sort SortOpt `json:"sort"`
|
|
|
|
// Username indicates if user names should be included in the results.
|
|
Username bool `json:"auth"`
|
|
|
|
// Subscriptions indicates if subscriptions should be included in the results.
|
|
Subscriptions bool `json:"subscriptions"`
|
|
|
|
// SubscriptionsDetail indicates if subscription details should be included in the results
|
|
SubscriptionsDetail bool `json:"subscriptions_detail"`
|
|
|
|
// Offset is used for pagination. Connz() only returns connections starting at this
|
|
// offset from the global results.
|
|
Offset int `json:"offset"`
|
|
|
|
// Limit is the maximum number of connections that should be returned by Connz().
|
|
Limit int `json:"limit"`
|
|
|
|
// Filter for this explicit client connection.
|
|
CID uint64 `json:"cid"`
|
|
|
|
// Filter for this explicit client connection based on the MQTT client ID
|
|
MQTTClient string `json:"mqtt_client"`
|
|
|
|
// Filter by connection state.
|
|
State ConnState `json:"state"`
|
|
|
|
// The below options only apply if auth is true.
|
|
|
|
// Filter by username.
|
|
User string `json:"user"`
|
|
|
|
// Filter by account.
|
|
Account string `json:"acc"`
|
|
|
|
// Filter by subject interest
|
|
FilterSubject string `json:"filter_subject"`
|
|
|
|
// Private indication that this request is from an account and not a system account.
|
|
// Used to not leak system level information to the account.
|
|
isAccountReq bool
|
|
}
|
|
|
|
// ConnState is for filtering states of connections. We will only have two, open and closed.
|
|
type ConnState int
|
|
|
|
const (
|
|
// ConnOpen filters on open clients.
|
|
ConnOpen = ConnState(iota)
|
|
// ConnClosed filters on closed clients.
|
|
ConnClosed
|
|
// ConnAll returns all clients.
|
|
ConnAll
|
|
)
|
|
|
|
// ConnInfo has detailed information on a per connection basis.
|
|
type ConnInfo struct {
|
|
Cid uint64 `json:"cid"`
|
|
Kind string `json:"kind,omitempty"`
|
|
Type string `json:"type,omitempty"`
|
|
IP string `json:"ip"`
|
|
Port int `json:"port"`
|
|
Start time.Time `json:"start"`
|
|
LastActivity time.Time `json:"last_activity"`
|
|
Stop *time.Time `json:"stop,omitempty"`
|
|
Reason string `json:"reason,omitempty"`
|
|
RTT string `json:"rtt,omitempty"`
|
|
Uptime string `json:"uptime"`
|
|
Idle string `json:"idle"`
|
|
Pending int `json:"pending_bytes"`
|
|
InMsgs int64 `json:"in_msgs"`
|
|
OutMsgs int64 `json:"out_msgs"`
|
|
InBytes int64 `json:"in_bytes"`
|
|
OutBytes int64 `json:"out_bytes"`
|
|
NumSubs uint32 `json:"subscriptions"`
|
|
Name string `json:"name,omitempty"`
|
|
Lang string `json:"lang,omitempty"`
|
|
Version string `json:"version,omitempty"`
|
|
TLSVersion string `json:"tls_version,omitempty"`
|
|
TLSCipher string `json:"tls_cipher_suite,omitempty"`
|
|
AuthorizedUser string `json:"authorized_user,omitempty"`
|
|
Account string `json:"account,omitempty"`
|
|
Subs []string `json:"subscriptions_list,omitempty"`
|
|
SubsDetail []SubDetail `json:"subscriptions_list_detail,omitempty"`
|
|
JWT string `json:"jwt,omitempty"`
|
|
IssuerKey string `json:"issuer_key,omitempty"`
|
|
NameTag string `json:"name_tag,omitempty"`
|
|
Tags jwt.TagList `json:"tags,omitempty"`
|
|
MQTTClient string `json:"mqtt_client,omitempty"` // This is the MQTT client id
|
|
}
|
|
|
|
// DefaultConnListSize is the default size of the connection list.
|
|
const DefaultConnListSize = 1024
|
|
|
|
// DefaultSubListSize is the default size of the subscriptions list.
|
|
const DefaultSubListSize = 1024
|
|
|
|
const defaultStackBufSize = 10000
|
|
|
|
func newSubsDetailList(client *client) []SubDetail {
|
|
subsDetail := make([]SubDetail, 0, len(client.subs))
|
|
for _, sub := range client.subs {
|
|
subsDetail = append(subsDetail, newClientSubDetail(sub))
|
|
}
|
|
return subsDetail
|
|
}
|
|
|
|
func newSubsList(client *client) []string {
|
|
subs := make([]string, 0, len(client.subs))
|
|
for _, sub := range client.subs {
|
|
subs = append(subs, string(sub.subject))
|
|
}
|
|
return subs
|
|
}
|
|
|
|
// Connz returns a Connz struct containing information about connections.
|
|
func (s *Server) Connz(opts *ConnzOptions) (*Connz, error) {
|
|
var (
|
|
sortOpt = ByCid
|
|
auth bool
|
|
subs bool
|
|
subsDet bool
|
|
offset int
|
|
limit = DefaultConnListSize
|
|
cid = uint64(0)
|
|
state = ConnOpen
|
|
user string
|
|
acc string
|
|
a *Account
|
|
filter string
|
|
mqttCID string
|
|
)
|
|
|
|
if opts != nil {
|
|
// If no sort option given or sort is by uptime, then sort by cid
|
|
if opts.Sort == _EMPTY_ {
|
|
sortOpt = ByCid
|
|
} else {
|
|
sortOpt = opts.Sort
|
|
if !sortOpt.IsValid() {
|
|
return nil, fmt.Errorf("invalid sorting option: %s", sortOpt)
|
|
}
|
|
}
|
|
|
|
// Auth specifics.
|
|
auth = opts.Username
|
|
if !auth && (user != _EMPTY_ || acc != _EMPTY_) {
|
|
return nil, fmt.Errorf("filter by user or account only allowed with auth option")
|
|
}
|
|
user = opts.User
|
|
acc = opts.Account
|
|
mqttCID = opts.MQTTClient
|
|
|
|
subs = opts.Subscriptions
|
|
subsDet = opts.SubscriptionsDetail
|
|
offset = opts.Offset
|
|
if offset < 0 {
|
|
offset = 0
|
|
}
|
|
limit = opts.Limit
|
|
if limit <= 0 {
|
|
limit = DefaultConnListSize
|
|
}
|
|
// state
|
|
state = opts.State
|
|
|
|
// ByStop only makes sense on closed connections
|
|
if sortOpt == ByStop && state != ConnClosed {
|
|
return nil, fmt.Errorf("sort by stop only valid on closed connections")
|
|
}
|
|
// ByReason is the same.
|
|
if sortOpt == ByReason && state != ConnClosed {
|
|
return nil, fmt.Errorf("sort by reason only valid on closed connections")
|
|
}
|
|
// If searching by CID
|
|
if opts.CID > 0 {
|
|
cid = opts.CID
|
|
limit = 1
|
|
}
|
|
// If filtering by subject.
|
|
if opts.FilterSubject != _EMPTY_ && opts.FilterSubject != fwcs {
|
|
if acc == _EMPTY_ {
|
|
return nil, fmt.Errorf("filter by subject only valid with account filtering")
|
|
}
|
|
filter = opts.FilterSubject
|
|
}
|
|
}
|
|
|
|
c := &Connz{
|
|
Offset: offset,
|
|
Limit: limit,
|
|
Now: time.Now().UTC(),
|
|
}
|
|
|
|
// Open clients
|
|
var openClients []*client
|
|
// Hold for closed clients if requested.
|
|
var closedClients []*closedClient
|
|
|
|
var clist map[uint64]*client
|
|
|
|
// If this is an account scoped request from a no $SYS account.
|
|
isAccReq := acc != _EMPTY_ && opts.isAccountReq
|
|
|
|
if acc != _EMPTY_ {
|
|
var err error
|
|
a, err = s.lookupAccount(acc)
|
|
if err != nil {
|
|
return c, nil
|
|
}
|
|
a.mu.RLock()
|
|
clist = make(map[uint64]*client, a.numLocalConnections())
|
|
for c := range a.clients {
|
|
if c.kind == CLIENT || c.kind == LEAF {
|
|
clist[c.cid] = c
|
|
}
|
|
}
|
|
a.mu.RUnlock()
|
|
}
|
|
|
|
// Walk the open client list with server lock held.
|
|
s.mu.Lock()
|
|
// Default to all client unless filled in above.
|
|
if clist == nil {
|
|
clist = s.clients
|
|
}
|
|
|
|
// copy the server id for monitoring
|
|
c.ID = s.info.ID
|
|
|
|
// Number of total clients. The resulting ConnInfo array
|
|
// may be smaller if pagination is used.
|
|
switch state {
|
|
case ConnOpen:
|
|
c.Total = len(clist)
|
|
case ConnClosed:
|
|
closedClients = s.closed.closedClients()
|
|
c.Total = len(closedClients)
|
|
case ConnAll:
|
|
c.Total = len(clist)
|
|
closedClients = s.closed.closedClients()
|
|
c.Total += len(closedClients)
|
|
}
|
|
|
|
// We may need to filter these connections.
|
|
if isAccReq && len(closedClients) > 0 {
|
|
var ccc []*closedClient
|
|
for _, cc := range closedClients {
|
|
if cc.acc == acc {
|
|
ccc = append(ccc, cc)
|
|
}
|
|
}
|
|
c.Total -= (len(closedClients) - len(ccc))
|
|
closedClients = ccc
|
|
}
|
|
|
|
totalClients := c.Total
|
|
if cid > 0 { // Meaning we only want 1.
|
|
totalClients = 1
|
|
}
|
|
if state == ConnOpen || state == ConnAll {
|
|
openClients = make([]*client, 0, totalClients)
|
|
}
|
|
|
|
// Data structures for results.
|
|
var conns []ConnInfo // Limits allocs for actual ConnInfos.
|
|
var pconns ConnInfos
|
|
|
|
switch state {
|
|
case ConnOpen:
|
|
conns = make([]ConnInfo, totalClients)
|
|
pconns = make(ConnInfos, totalClients)
|
|
case ConnClosed:
|
|
pconns = make(ConnInfos, totalClients)
|
|
case ConnAll:
|
|
conns = make([]ConnInfo, cap(openClients))
|
|
pconns = make(ConnInfos, totalClients)
|
|
}
|
|
|
|
// Search by individual CID.
|
|
if cid > 0 {
|
|
if state == ConnClosed || state == ConnAll {
|
|
copyClosed := closedClients
|
|
closedClients = nil
|
|
for _, cc := range copyClosed {
|
|
if cc.Cid == cid {
|
|
closedClients = []*closedClient{cc}
|
|
break
|
|
}
|
|
}
|
|
} else if state == ConnOpen || state == ConnAll {
|
|
client := s.clients[cid]
|
|
if client != nil {
|
|
openClients = append(openClients, client)
|
|
}
|
|
}
|
|
} else {
|
|
// Gather all open clients.
|
|
if state == ConnOpen || state == ConnAll {
|
|
for _, client := range clist {
|
|
// If we have an account specified we need to filter.
|
|
if acc != _EMPTY_ && (client.acc == nil || client.acc.Name != acc) {
|
|
continue
|
|
}
|
|
// Do user filtering second
|
|
if user != _EMPTY_ && client.opts.Username != user {
|
|
continue
|
|
}
|
|
// Do mqtt client ID filtering next
|
|
if mqttCID != _EMPTY_ && client.getMQTTClientID() != mqttCID {
|
|
continue
|
|
}
|
|
openClients = append(openClients, client)
|
|
}
|
|
}
|
|
}
|
|
s.mu.Unlock()
|
|
|
|
// Filter by subject now if needed. We do this outside of server lock.
|
|
if filter != _EMPTY_ {
|
|
var oc []*client
|
|
for _, c := range openClients {
|
|
c.mu.Lock()
|
|
for _, sub := range c.subs {
|
|
if SubjectsCollide(filter, string(sub.subject)) {
|
|
oc = append(oc, c)
|
|
break
|
|
}
|
|
}
|
|
c.mu.Unlock()
|
|
openClients = oc
|
|
}
|
|
}
|
|
|
|
// Just return with empty array if nothing here.
|
|
if len(openClients) == 0 && len(closedClients) == 0 {
|
|
c.Conns = ConnInfos{}
|
|
return c, nil
|
|
}
|
|
|
|
// Now whip through and generate ConnInfo entries
|
|
// Open Clients
|
|
i := 0
|
|
for _, client := range openClients {
|
|
client.mu.Lock()
|
|
ci := &conns[i]
|
|
ci.fill(client, client.nc, c.Now)
|
|
// Fill in subscription data if requested.
|
|
if len(client.subs) > 0 {
|
|
if subsDet {
|
|
ci.SubsDetail = newSubsDetailList(client)
|
|
} else if subs {
|
|
ci.Subs = newSubsList(client)
|
|
}
|
|
}
|
|
// Fill in user if auth requested.
|
|
if auth {
|
|
ci.AuthorizedUser = client.getRawAuthUser()
|
|
// Add in account iff not the global account.
|
|
if client.acc != nil && (client.acc.Name != globalAccountName) {
|
|
ci.Account = client.acc.Name
|
|
}
|
|
ci.JWT = client.opts.JWT
|
|
ci.IssuerKey = issuerForClient(client)
|
|
ci.Tags = client.tags
|
|
ci.NameTag = client.nameTag
|
|
}
|
|
client.mu.Unlock()
|
|
pconns[i] = ci
|
|
i++
|
|
}
|
|
// Closed Clients
|
|
var needCopy bool
|
|
if subs || auth {
|
|
needCopy = true
|
|
}
|
|
for _, cc := range closedClients {
|
|
// If we have an account specified we need to filter.
|
|
if acc != _EMPTY_ && cc.acc != acc {
|
|
continue
|
|
}
|
|
// Do user filtering second
|
|
if user != _EMPTY_ && cc.user != user {
|
|
continue
|
|
}
|
|
// Do mqtt client ID filtering next
|
|
if mqttCID != _EMPTY_ && cc.MQTTClient != mqttCID {
|
|
continue
|
|
}
|
|
// Copy if needed for any changes to the ConnInfo
|
|
if needCopy {
|
|
cx := *cc
|
|
cc = &cx
|
|
}
|
|
// Fill in subscription data if requested.
|
|
if len(cc.subs) > 0 {
|
|
if subsDet {
|
|
cc.SubsDetail = cc.subs
|
|
} else if subs {
|
|
cc.Subs = make([]string, 0, len(cc.subs))
|
|
for _, sub := range cc.subs {
|
|
cc.Subs = append(cc.Subs, sub.Subject)
|
|
}
|
|
}
|
|
}
|
|
// Fill in user if auth requested.
|
|
if auth {
|
|
cc.AuthorizedUser = cc.user
|
|
// Add in account iff not the global account.
|
|
if cc.acc != "" && (cc.acc != globalAccountName) {
|
|
cc.Account = cc.acc
|
|
}
|
|
}
|
|
pconns[i] = &cc.ConnInfo
|
|
i++
|
|
}
|
|
|
|
// This will trip if we have filtered out client connections.
|
|
if len(pconns) != i {
|
|
pconns = pconns[:i]
|
|
totalClients = i
|
|
}
|
|
|
|
switch sortOpt {
|
|
case ByCid, ByStart:
|
|
sort.Sort(byCid{pconns})
|
|
case BySubs:
|
|
sort.Sort(sort.Reverse(bySubs{pconns}))
|
|
case ByPending:
|
|
sort.Sort(sort.Reverse(byPending{pconns}))
|
|
case ByOutMsgs:
|
|
sort.Sort(sort.Reverse(byOutMsgs{pconns}))
|
|
case ByInMsgs:
|
|
sort.Sort(sort.Reverse(byInMsgs{pconns}))
|
|
case ByOutBytes:
|
|
sort.Sort(sort.Reverse(byOutBytes{pconns}))
|
|
case ByInBytes:
|
|
sort.Sort(sort.Reverse(byInBytes{pconns}))
|
|
case ByLast:
|
|
sort.Sort(sort.Reverse(byLast{pconns}))
|
|
case ByIdle:
|
|
sort.Sort(sort.Reverse(byIdle{pconns}))
|
|
case ByUptime:
|
|
sort.Sort(byUptime{pconns, time.Now()})
|
|
case ByStop:
|
|
sort.Sort(sort.Reverse(byStop{pconns}))
|
|
case ByReason:
|
|
sort.Sort(byReason{pconns})
|
|
}
|
|
|
|
minoff := c.Offset
|
|
maxoff := c.Offset + c.Limit
|
|
|
|
maxIndex := totalClients
|
|
|
|
// Make sure these are sane.
|
|
if minoff > maxIndex {
|
|
minoff = maxIndex
|
|
}
|
|
if maxoff > maxIndex {
|
|
maxoff = maxIndex
|
|
}
|
|
|
|
// Now pare down to the requested size.
|
|
// TODO(dlc) - for very large number of connections we
|
|
// could save the whole list in a hash, send hash on first
|
|
// request and allow users to use has for subsequent pages.
|
|
// Low TTL, say < 1sec.
|
|
c.Conns = pconns[minoff:maxoff]
|
|
c.NumConns = len(c.Conns)
|
|
|
|
return c, nil
|
|
}
|
|
|
|
// Fills in the ConnInfo from the client.
|
|
// client should be locked.
|
|
func (ci *ConnInfo) fill(client *client, nc net.Conn, now time.Time) {
|
|
ci.Cid = client.cid
|
|
ci.MQTTClient = client.getMQTTClientID()
|
|
ci.Kind = client.kindString()
|
|
ci.Type = client.clientTypeString()
|
|
ci.Start = client.start
|
|
ci.LastActivity = client.last
|
|
ci.Uptime = myUptime(now.Sub(client.start))
|
|
ci.Idle = myUptime(now.Sub(client.last))
|
|
ci.RTT = client.getRTT().String()
|
|
ci.OutMsgs = client.outMsgs
|
|
ci.OutBytes = client.outBytes
|
|
ci.NumSubs = uint32(len(client.subs))
|
|
ci.Pending = int(client.out.pb)
|
|
ci.Name = client.opts.Name
|
|
ci.Lang = client.opts.Lang
|
|
ci.Version = client.opts.Version
|
|
// inMsgs and inBytes are updated outside of the client's lock, so
|
|
// we need to use atomic here.
|
|
ci.InMsgs = atomic.LoadInt64(&client.inMsgs)
|
|
ci.InBytes = atomic.LoadInt64(&client.inBytes)
|
|
|
|
// If the connection is gone, too bad, we won't set TLSVersion and TLSCipher.
|
|
// Exclude clients that are still doing handshake so we don't block in
|
|
// ConnectionState().
|
|
if client.flags.isSet(handshakeComplete) && nc != nil {
|
|
conn := nc.(*tls.Conn)
|
|
cs := conn.ConnectionState()
|
|
ci.TLSVersion = tlsVersion(cs.Version)
|
|
ci.TLSCipher = tlsCipher(cs.CipherSuite)
|
|
}
|
|
|
|
if client.port != 0 {
|
|
ci.Port = int(client.port)
|
|
ci.IP = client.host
|
|
}
|
|
}
|
|
|
|
// Assume lock is held
|
|
func (c *client) getRTT() time.Duration {
|
|
if c.rtt == 0 {
|
|
// If a real client, go ahead and send ping now to get a value
|
|
// for RTT. For tests and telnet, or if client is closing, etc skip.
|
|
if c.opts.Lang != "" {
|
|
c.sendRTTPingLocked()
|
|
}
|
|
return 0
|
|
}
|
|
var rtt time.Duration
|
|
if c.rtt > time.Microsecond && c.rtt < time.Millisecond {
|
|
rtt = c.rtt.Truncate(time.Microsecond)
|
|
} else {
|
|
rtt = c.rtt.Truncate(time.Nanosecond)
|
|
}
|
|
return rtt
|
|
}
|
|
|
|
func decodeBool(w http.ResponseWriter, r *http.Request, param string) (bool, error) {
|
|
str := r.URL.Query().Get(param)
|
|
if str == "" {
|
|
return false, nil
|
|
}
|
|
val, err := strconv.ParseBool(str)
|
|
if err != nil {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
w.Write([]byte(fmt.Sprintf("Error decoding boolean for '%s': %v", param, err)))
|
|
return false, err
|
|
}
|
|
return val, nil
|
|
}
|
|
|
|
func decodeUint64(w http.ResponseWriter, r *http.Request, param string) (uint64, error) {
|
|
str := r.URL.Query().Get(param)
|
|
if str == "" {
|
|
return 0, nil
|
|
}
|
|
val, err := strconv.ParseUint(str, 10, 64)
|
|
if err != nil {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
w.Write([]byte(fmt.Sprintf("Error decoding uint64 for '%s': %v", param, err)))
|
|
return 0, err
|
|
}
|
|
return val, nil
|
|
}
|
|
|
|
func decodeInt(w http.ResponseWriter, r *http.Request, param string) (int, error) {
|
|
str := r.URL.Query().Get(param)
|
|
if str == "" {
|
|
return 0, nil
|
|
}
|
|
val, err := strconv.Atoi(str)
|
|
if err != nil {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
w.Write([]byte(fmt.Sprintf("Error decoding int for '%s': %v", param, err)))
|
|
return 0, err
|
|
}
|
|
return val, nil
|
|
}
|
|
|
|
func decodeState(w http.ResponseWriter, r *http.Request) (ConnState, error) {
|
|
str := r.URL.Query().Get("state")
|
|
if str == "" {
|
|
return ConnOpen, nil
|
|
}
|
|
switch strings.ToLower(str) {
|
|
case "open":
|
|
return ConnOpen, nil
|
|
case "closed":
|
|
return ConnClosed, nil
|
|
case "any", "all":
|
|
return ConnAll, nil
|
|
}
|
|
// We do not understand intended state here.
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
err := fmt.Errorf("Error decoding state for %s", str)
|
|
w.Write([]byte(err.Error()))
|
|
return 0, err
|
|
}
|
|
|
|
func decodeSubs(w http.ResponseWriter, r *http.Request) (subs bool, subsDet bool, err error) {
|
|
subsDet = strings.ToLower(r.URL.Query().Get("subs")) == "detail"
|
|
if !subsDet {
|
|
subs, err = decodeBool(w, r, "subs")
|
|
}
|
|
return
|
|
}
|
|
|
|
// HandleConnz process HTTP requests for connection information.
|
|
func (s *Server) HandleConnz(w http.ResponseWriter, r *http.Request) {
|
|
sortOpt := SortOpt(r.URL.Query().Get("sort"))
|
|
auth, err := decodeBool(w, r, "auth")
|
|
if err != nil {
|
|
return
|
|
}
|
|
subs, subsDet, err := decodeSubs(w, r)
|
|
if err != nil {
|
|
return
|
|
}
|
|
offset, err := decodeInt(w, r, "offset")
|
|
if err != nil {
|
|
return
|
|
}
|
|
limit, err := decodeInt(w, r, "limit")
|
|
if err != nil {
|
|
return
|
|
}
|
|
cid, err := decodeUint64(w, r, "cid")
|
|
if err != nil {
|
|
return
|
|
}
|
|
state, err := decodeState(w, r)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
user := r.URL.Query().Get("user")
|
|
acc := r.URL.Query().Get("acc")
|
|
mqttCID := r.URL.Query().Get("mqtt_client")
|
|
|
|
connzOpts := &ConnzOptions{
|
|
Sort: sortOpt,
|
|
Username: auth,
|
|
Subscriptions: subs,
|
|
SubscriptionsDetail: subsDet,
|
|
Offset: offset,
|
|
Limit: limit,
|
|
CID: cid,
|
|
MQTTClient: mqttCID,
|
|
State: state,
|
|
User: user,
|
|
Account: acc,
|
|
}
|
|
|
|
s.mu.Lock()
|
|
s.httpReqStats[ConnzPath]++
|
|
s.mu.Unlock()
|
|
|
|
c, err := s.Connz(connzOpts)
|
|
if err != nil {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
w.Write([]byte(err.Error()))
|
|
return
|
|
}
|
|
b, err := json.MarshalIndent(c, "", " ")
|
|
if err != nil {
|
|
s.Errorf("Error marshaling response to /connz request: %v", err)
|
|
}
|
|
|
|
// Handle response
|
|
ResponseHandler(w, r, b)
|
|
}
|
|
|
|
// Routez represents detailed information on current client connections.
|
|
type Routez struct {
|
|
ID string `json:"server_id"`
|
|
Now time.Time `json:"now"`
|
|
Import *SubjectPermission `json:"import,omitempty"`
|
|
Export *SubjectPermission `json:"export,omitempty"`
|
|
NumRoutes int `json:"num_routes"`
|
|
Routes []*RouteInfo `json:"routes"`
|
|
}
|
|
|
|
// RoutezOptions are options passed to Routez
|
|
type RoutezOptions struct {
|
|
// Subscriptions indicates that Routez will return a route's subscriptions
|
|
Subscriptions bool `json:"subscriptions"`
|
|
// SubscriptionsDetail indicates if subscription details should be included in the results
|
|
SubscriptionsDetail bool `json:"subscriptions_detail"`
|
|
}
|
|
|
|
// RouteInfo has detailed information on a per connection basis.
|
|
type RouteInfo struct {
|
|
Rid uint64 `json:"rid"`
|
|
RemoteID string `json:"remote_id"`
|
|
DidSolicit bool `json:"did_solicit"`
|
|
IsConfigured bool `json:"is_configured"`
|
|
IP string `json:"ip"`
|
|
Port int `json:"port"`
|
|
Import *SubjectPermission `json:"import,omitempty"`
|
|
Export *SubjectPermission `json:"export,omitempty"`
|
|
Pending int `json:"pending_size"`
|
|
RTT string `json:"rtt,omitempty"`
|
|
InMsgs int64 `json:"in_msgs"`
|
|
OutMsgs int64 `json:"out_msgs"`
|
|
InBytes int64 `json:"in_bytes"`
|
|
OutBytes int64 `json:"out_bytes"`
|
|
NumSubs uint32 `json:"subscriptions"`
|
|
Subs []string `json:"subscriptions_list,omitempty"`
|
|
SubsDetail []SubDetail `json:"subscriptions_list_detail,omitempty"`
|
|
}
|
|
|
|
// Routez returns a Routez struct containing information about routes.
|
|
func (s *Server) Routez(routezOpts *RoutezOptions) (*Routez, error) {
|
|
rs := &Routez{Routes: []*RouteInfo{}}
|
|
rs.Now = time.Now().UTC()
|
|
|
|
if routezOpts == nil {
|
|
routezOpts = &RoutezOptions{}
|
|
}
|
|
|
|
s.mu.Lock()
|
|
rs.NumRoutes = len(s.routes)
|
|
|
|
// copy the server id for monitoring
|
|
rs.ID = s.info.ID
|
|
|
|
// Check for defined permissions for all connected routes.
|
|
if perms := s.getOpts().Cluster.Permissions; perms != nil {
|
|
rs.Import = perms.Import
|
|
rs.Export = perms.Export
|
|
}
|
|
|
|
// Walk the list
|
|
for _, r := range s.routes {
|
|
r.mu.Lock()
|
|
ri := &RouteInfo{
|
|
Rid: r.cid,
|
|
RemoteID: r.route.remoteID,
|
|
DidSolicit: r.route.didSolicit,
|
|
IsConfigured: r.route.routeType == Explicit,
|
|
InMsgs: atomic.LoadInt64(&r.inMsgs),
|
|
OutMsgs: r.outMsgs,
|
|
InBytes: atomic.LoadInt64(&r.inBytes),
|
|
OutBytes: r.outBytes,
|
|
NumSubs: uint32(len(r.subs)),
|
|
Import: r.opts.Import,
|
|
Export: r.opts.Export,
|
|
RTT: r.getRTT().String(),
|
|
}
|
|
|
|
if len(r.subs) > 0 {
|
|
if routezOpts.SubscriptionsDetail {
|
|
ri.SubsDetail = newSubsDetailList(r)
|
|
} else if routezOpts.Subscriptions {
|
|
ri.Subs = newSubsList(r)
|
|
}
|
|
}
|
|
|
|
switch conn := r.nc.(type) {
|
|
case *net.TCPConn, *tls.Conn:
|
|
addr := conn.RemoteAddr().(*net.TCPAddr)
|
|
ri.Port = addr.Port
|
|
ri.IP = addr.IP.String()
|
|
}
|
|
r.mu.Unlock()
|
|
rs.Routes = append(rs.Routes, ri)
|
|
}
|
|
s.mu.Unlock()
|
|
return rs, nil
|
|
}
|
|
|
|
// HandleRoutez process HTTP requests for route information.
|
|
func (s *Server) HandleRoutez(w http.ResponseWriter, r *http.Request) {
|
|
subs, subsDetail, err := decodeSubs(w, r)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
opts := RoutezOptions{Subscriptions: subs, SubscriptionsDetail: subsDetail}
|
|
|
|
s.mu.Lock()
|
|
s.httpReqStats[RoutezPath]++
|
|
s.mu.Unlock()
|
|
|
|
// As of now, no error is ever returned.
|
|
rs, _ := s.Routez(&opts)
|
|
b, err := json.MarshalIndent(rs, "", " ")
|
|
if err != nil {
|
|
s.Errorf("Error marshaling response to /routez request: %v", err)
|
|
}
|
|
|
|
// Handle response
|
|
ResponseHandler(w, r, b)
|
|
}
|
|
|
|
// Subsz represents detail information on current connections.
|
|
type Subsz struct {
|
|
ID string `json:"server_id"`
|
|
Now time.Time `json:"now"`
|
|
*SublistStats
|
|
Total int `json:"total"`
|
|
Offset int `json:"offset"`
|
|
Limit int `json:"limit"`
|
|
Subs []SubDetail `json:"subscriptions_list,omitempty"`
|
|
}
|
|
|
|
// SubszOptions are the options passed to Subsz.
|
|
// As of now, there are no options defined.
|
|
type SubszOptions struct {
|
|
// Offset is used for pagination. Subsz() only returns connections starting at this
|
|
// offset from the global results.
|
|
Offset int `json:"offset"`
|
|
|
|
// Limit is the maximum number of subscriptions that should be returned by Subsz().
|
|
Limit int `json:"limit"`
|
|
|
|
// Subscriptions indicates if subscription details should be included in the results.
|
|
Subscriptions bool `json:"subscriptions"`
|
|
|
|
// Filter based on this account name.
|
|
Account string `json:"account,omitempty"`
|
|
|
|
// Test the list against this subject. Needs to be literal since it signifies a publish subject.
|
|
// We will only return subscriptions that would match if a message was sent to this subject.
|
|
Test string `json:"test,omitempty"`
|
|
}
|
|
|
|
// SubDetail is for verbose information for subscriptions.
|
|
type SubDetail struct {
|
|
Account string `json:"account,omitempty"`
|
|
Subject string `json:"subject"`
|
|
Queue string `json:"qgroup,omitempty"`
|
|
Sid string `json:"sid"`
|
|
Msgs int64 `json:"msgs"`
|
|
Max int64 `json:"max,omitempty"`
|
|
Cid uint64 `json:"cid"`
|
|
}
|
|
|
|
// Subscription client should be locked and guaranteed to be present.
|
|
func newSubDetail(sub *subscription) SubDetail {
|
|
sd := newClientSubDetail(sub)
|
|
if sub.client.acc != nil {
|
|
sd.Account = sub.client.acc.Name
|
|
}
|
|
return sd
|
|
}
|
|
|
|
// For subs details under clients.
|
|
func newClientSubDetail(sub *subscription) SubDetail {
|
|
return SubDetail{
|
|
Subject: string(sub.subject),
|
|
Queue: string(sub.queue),
|
|
Sid: string(sub.sid),
|
|
Msgs: sub.nm,
|
|
Max: sub.max,
|
|
Cid: sub.client.cid,
|
|
}
|
|
}
|
|
|
|
// Subsz returns a Subsz struct containing subjects statistics
|
|
func (s *Server) Subsz(opts *SubszOptions) (*Subsz, error) {
|
|
var (
|
|
subdetail bool
|
|
test bool
|
|
offset int
|
|
limit = DefaultSubListSize
|
|
testSub = ""
|
|
filterAcc = ""
|
|
)
|
|
|
|
if opts != nil {
|
|
subdetail = opts.Subscriptions
|
|
offset = opts.Offset
|
|
if offset < 0 {
|
|
offset = 0
|
|
}
|
|
limit = opts.Limit
|
|
if limit <= 0 {
|
|
limit = DefaultSubListSize
|
|
}
|
|
if opts.Test != "" {
|
|
testSub = opts.Test
|
|
test = true
|
|
if !IsValidLiteralSubject(testSub) {
|
|
return nil, fmt.Errorf("invalid test subject, must be valid publish subject: %s", testSub)
|
|
}
|
|
}
|
|
if opts.Account != "" {
|
|
filterAcc = opts.Account
|
|
}
|
|
}
|
|
|
|
slStats := &SublistStats{}
|
|
|
|
// FIXME(dlc) - Make account aware.
|
|
sz := &Subsz{s.info.ID, time.Now().UTC(), slStats, 0, offset, limit, nil}
|
|
|
|
if subdetail {
|
|
var raw [4096]*subscription
|
|
subs := raw[:0]
|
|
s.accounts.Range(func(k, v interface{}) bool {
|
|
acc := v.(*Account)
|
|
if filterAcc != "" && acc.GetName() != filterAcc {
|
|
return true
|
|
}
|
|
slStats.add(acc.sl.Stats())
|
|
acc.sl.localSubs(&subs, false)
|
|
return true
|
|
})
|
|
|
|
details := make([]SubDetail, len(subs))
|
|
i := 0
|
|
// TODO(dlc) - may be inefficient and could just do normal match when total subs is large and filtering.
|
|
for _, sub := range subs {
|
|
// Check for filter
|
|
if test && !matchLiteral(testSub, string(sub.subject)) {
|
|
continue
|
|
}
|
|
if sub.client == nil {
|
|
continue
|
|
}
|
|
sub.client.mu.Lock()
|
|
details[i] = newSubDetail(sub)
|
|
sub.client.mu.Unlock()
|
|
i++
|
|
}
|
|
minoff := sz.Offset
|
|
maxoff := sz.Offset + sz.Limit
|
|
|
|
maxIndex := i
|
|
|
|
// Make sure these are sane.
|
|
if minoff > maxIndex {
|
|
minoff = maxIndex
|
|
}
|
|
if maxoff > maxIndex {
|
|
maxoff = maxIndex
|
|
}
|
|
sz.Subs = details[minoff:maxoff]
|
|
sz.Total = len(sz.Subs)
|
|
} else {
|
|
s.accounts.Range(func(k, v interface{}) bool {
|
|
acc := v.(*Account)
|
|
if filterAcc != "" && acc.GetName() != filterAcc {
|
|
return true
|
|
}
|
|
slStats.add(acc.sl.Stats())
|
|
return true
|
|
})
|
|
}
|
|
|
|
return sz, nil
|
|
}
|
|
|
|
// HandleSubsz processes HTTP requests for subjects stats.
|
|
func (s *Server) HandleSubsz(w http.ResponseWriter, r *http.Request) {
|
|
s.mu.Lock()
|
|
s.httpReqStats[SubszPath]++
|
|
s.mu.Unlock()
|
|
|
|
subs, err := decodeBool(w, r, "subs")
|
|
if err != nil {
|
|
return
|
|
}
|
|
offset, err := decodeInt(w, r, "offset")
|
|
if err != nil {
|
|
return
|
|
}
|
|
limit, err := decodeInt(w, r, "limit")
|
|
if err != nil {
|
|
return
|
|
}
|
|
testSub := r.URL.Query().Get("test")
|
|
// Filtered account.
|
|
filterAcc := r.URL.Query().Get("acc")
|
|
|
|
subszOpts := &SubszOptions{
|
|
Subscriptions: subs,
|
|
Offset: offset,
|
|
Limit: limit,
|
|
Account: filterAcc,
|
|
Test: testSub,
|
|
}
|
|
|
|
st, err := s.Subsz(subszOpts)
|
|
if err != nil {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
w.Write([]byte(err.Error()))
|
|
return
|
|
}
|
|
|
|
var b []byte
|
|
|
|
if len(st.Subs) == 0 {
|
|
b, err = json.MarshalIndent(st.SublistStats, "", " ")
|
|
} else {
|
|
b, err = json.MarshalIndent(st, "", " ")
|
|
}
|
|
if err != nil {
|
|
s.Errorf("Error marshaling response to /subscriptionsz request: %v", err)
|
|
}
|
|
|
|
// Handle response
|
|
ResponseHandler(w, r, b)
|
|
}
|
|
|
|
// HandleStacksz processes HTTP requests for getting stacks
|
|
func (s *Server) HandleStacksz(w http.ResponseWriter, r *http.Request) {
|
|
// Do not get any lock here that would prevent getting the stacks
|
|
// if we were to have a deadlock somewhere.
|
|
var defaultBuf [defaultStackBufSize]byte
|
|
size := defaultStackBufSize
|
|
buf := defaultBuf[:size]
|
|
n := 0
|
|
for {
|
|
n = runtime.Stack(buf, true)
|
|
if n < size {
|
|
break
|
|
}
|
|
size *= 2
|
|
buf = make([]byte, size)
|
|
}
|
|
// Handle response
|
|
ResponseHandler(w, r, buf[:n])
|
|
}
|
|
|
|
type monitorIPQueue struct {
|
|
Pending int `json:"pending"`
|
|
InProgress int `json:"in_progress,omitempty"`
|
|
}
|
|
|
|
func (s *Server) HandleIPQueuesz(w http.ResponseWriter, r *http.Request) {
|
|
all, err := decodeBool(w, r, "all")
|
|
if err != nil {
|
|
return
|
|
}
|
|
qfilter := r.URL.Query().Get("queues")
|
|
|
|
queues := map[string]monitorIPQueue{}
|
|
|
|
s.ipQueues.Range(func(k, v interface{}) bool {
|
|
name := k.(string)
|
|
queue := v.(*ipQueue)
|
|
pending := queue.len()
|
|
inProgress := int(queue.inProgress())
|
|
if !all && (pending == 0 && inProgress == 0) {
|
|
return true
|
|
} else if qfilter != _EMPTY_ && !strings.Contains(name, qfilter) {
|
|
return true
|
|
}
|
|
queues[name] = monitorIPQueue{Pending: pending, InProgress: inProgress}
|
|
return true
|
|
})
|
|
|
|
b, _ := json.MarshalIndent(queues, "", " ")
|
|
ResponseHandler(w, r, b)
|
|
}
|
|
|
|
// Varz will output server information on the monitoring port at /varz.
|
|
type Varz struct {
|
|
ID string `json:"server_id"`
|
|
Name string `json:"server_name"`
|
|
Version string `json:"version"`
|
|
Proto int `json:"proto"`
|
|
GitCommit string `json:"git_commit,omitempty"`
|
|
GoVersion string `json:"go"`
|
|
Host string `json:"host"`
|
|
Port int `json:"port"`
|
|
AuthRequired bool `json:"auth_required,omitempty"`
|
|
TLSRequired bool `json:"tls_required,omitempty"`
|
|
TLSVerify bool `json:"tls_verify,omitempty"`
|
|
IP string `json:"ip,omitempty"`
|
|
ClientConnectURLs []string `json:"connect_urls,omitempty"`
|
|
WSConnectURLs []string `json:"ws_connect_urls,omitempty"`
|
|
MaxConn int `json:"max_connections"`
|
|
MaxSubs int `json:"max_subscriptions,omitempty"`
|
|
PingInterval time.Duration `json:"ping_interval"`
|
|
MaxPingsOut int `json:"ping_max"`
|
|
HTTPHost string `json:"http_host"`
|
|
HTTPPort int `json:"http_port"`
|
|
HTTPBasePath string `json:"http_base_path"`
|
|
HTTPSPort int `json:"https_port"`
|
|
AuthTimeout float64 `json:"auth_timeout"`
|
|
MaxControlLine int32 `json:"max_control_line"`
|
|
MaxPayload int `json:"max_payload"`
|
|
MaxPending int64 `json:"max_pending"`
|
|
Cluster ClusterOptsVarz `json:"cluster,omitempty"`
|
|
Gateway GatewayOptsVarz `json:"gateway,omitempty"`
|
|
LeafNode LeafNodeOptsVarz `json:"leaf,omitempty"`
|
|
MQTT MQTTOptsVarz `json:"mqtt,omitempty"`
|
|
Websocket WebsocketOptsVarz `json:"websocket,omitempty"`
|
|
JetStream JetStreamVarz `json:"jetstream,omitempty"`
|
|
TLSTimeout float64 `json:"tls_timeout"`
|
|
WriteDeadline time.Duration `json:"write_deadline"`
|
|
Start time.Time `json:"start"`
|
|
Now time.Time `json:"now"`
|
|
Uptime string `json:"uptime"`
|
|
Mem int64 `json:"mem"`
|
|
Cores int `json:"cores"`
|
|
MaxProcs int `json:"gomaxprocs"`
|
|
CPU float64 `json:"cpu"`
|
|
Connections int `json:"connections"`
|
|
TotalConnections uint64 `json:"total_connections"`
|
|
Routes int `json:"routes"`
|
|
Remotes int `json:"remotes"`
|
|
Leafs int `json:"leafnodes"`
|
|
InMsgs int64 `json:"in_msgs"`
|
|
OutMsgs int64 `json:"out_msgs"`
|
|
InBytes int64 `json:"in_bytes"`
|
|
OutBytes int64 `json:"out_bytes"`
|
|
SlowConsumers int64 `json:"slow_consumers"`
|
|
Subscriptions uint32 `json:"subscriptions"`
|
|
HTTPReqStats map[string]uint64 `json:"http_req_stats"`
|
|
ConfigLoadTime time.Time `json:"config_load_time"`
|
|
Tags jwt.TagList `json:"tags,omitempty"`
|
|
TrustedOperatorsJwt []string `json:"trusted_operators_jwt,omitempty"`
|
|
TrustedOperatorsClaim []*jwt.OperatorClaims `json:"trusted_operators_claim,omitempty"`
|
|
SystemAccount string `json:"system_account,omitempty"`
|
|
PinnedAccountFail uint64 `json:"pinned_account_fails,omitempty"`
|
|
}
|
|
|
|
// JetStreamVarz contains basic runtime information about jetstream
|
|
type JetStreamVarz struct {
|
|
Config *JetStreamConfig `json:"config,omitempty"`
|
|
Stats *JetStreamStats `json:"stats,omitempty"`
|
|
Meta *MetaClusterInfo `json:"meta,omitempty"`
|
|
}
|
|
|
|
// ClusterOptsVarz contains monitoring cluster information
|
|
type ClusterOptsVarz struct {
|
|
Name string `json:"name,omitempty"`
|
|
Host string `json:"addr,omitempty"`
|
|
Port int `json:"cluster_port,omitempty"`
|
|
AuthTimeout float64 `json:"auth_timeout,omitempty"`
|
|
URLs []string `json:"urls,omitempty"`
|
|
TLSTimeout float64 `json:"tls_timeout,omitempty"`
|
|
TLSRequired bool `json:"tls_required,omitempty"`
|
|
TLSVerify bool `json:"tls_verify,omitempty"`
|
|
}
|
|
|
|
// GatewayOptsVarz contains monitoring gateway information
|
|
type GatewayOptsVarz struct {
|
|
Name string `json:"name,omitempty"`
|
|
Host string `json:"host,omitempty"`
|
|
Port int `json:"port,omitempty"`
|
|
AuthTimeout float64 `json:"auth_timeout,omitempty"`
|
|
TLSTimeout float64 `json:"tls_timeout,omitempty"`
|
|
TLSRequired bool `json:"tls_required,omitempty"`
|
|
TLSVerify bool `json:"tls_verify,omitempty"`
|
|
Advertise string `json:"advertise,omitempty"`
|
|
ConnectRetries int `json:"connect_retries,omitempty"`
|
|
Gateways []RemoteGatewayOptsVarz `json:"gateways,omitempty"`
|
|
RejectUnknown bool `json:"reject_unknown,omitempty"` // config got renamed to reject_unknown_cluster
|
|
}
|
|
|
|
// RemoteGatewayOptsVarz contains monitoring remote gateway information
|
|
type RemoteGatewayOptsVarz struct {
|
|
Name string `json:"name"`
|
|
TLSTimeout float64 `json:"tls_timeout,omitempty"`
|
|
URLs []string `json:"urls,omitempty"`
|
|
}
|
|
|
|
// LeafNodeOptsVarz contains monitoring leaf node information
|
|
type LeafNodeOptsVarz struct {
|
|
Host string `json:"host,omitempty"`
|
|
Port int `json:"port,omitempty"`
|
|
AuthTimeout float64 `json:"auth_timeout,omitempty"`
|
|
TLSTimeout float64 `json:"tls_timeout,omitempty"`
|
|
TLSRequired bool `json:"tls_required,omitempty"`
|
|
TLSVerify bool `json:"tls_verify,omitempty"`
|
|
Remotes []RemoteLeafOptsVarz `json:"remotes,omitempty"`
|
|
}
|
|
|
|
// DenyRules Contains lists of subjects not allowed to be imported/exported
|
|
type DenyRules struct {
|
|
Exports []string `json:"exports,omitempty"`
|
|
Imports []string `json:"imports,omitempty"`
|
|
}
|
|
|
|
// RemoteLeafOptsVarz contains monitoring remote leaf node information
|
|
type RemoteLeafOptsVarz struct {
|
|
LocalAccount string `json:"local_account,omitempty"`
|
|
TLSTimeout float64 `json:"tls_timeout,omitempty"`
|
|
URLs []string `json:"urls,omitempty"`
|
|
Deny *DenyRules `json:"deny,omitempty"`
|
|
}
|
|
|
|
// MQTTOptsVarz contains monitoring MQTT information
|
|
type MQTTOptsVarz struct {
|
|
Host string `json:"host,omitempty"`
|
|
Port int `json:"port,omitempty"`
|
|
NoAuthUser string `json:"no_auth_user,omitempty"`
|
|
AuthTimeout float64 `json:"auth_timeout,omitempty"`
|
|
TLSMap bool `json:"tls_map,omitempty"`
|
|
TLSTimeout float64 `json:"tls_timeout,omitempty"`
|
|
TLSPinnedCerts []string `json:"tls_pinned_certs,omitempty"`
|
|
JsDomain string `json:"js_domain,omitempty"`
|
|
AckWait time.Duration `json:"ack_wait,omitempty"`
|
|
MaxAckPending uint16 `json:"max_ack_pending,omitempty"`
|
|
}
|
|
|
|
// WebsocketOptsVarz contains monitoring websocket information
|
|
type WebsocketOptsVarz struct {
|
|
Host string `json:"host,omitempty"`
|
|
Port int `json:"port,omitempty"`
|
|
Advertise string `json:"advertise,omitempty"`
|
|
NoAuthUser string `json:"no_auth_user,omitempty"`
|
|
JWTCookie string `json:"jwt_cookie,omitempty"`
|
|
HandshakeTimeout time.Duration `json:"handshake_timeout,omitempty"`
|
|
AuthTimeout float64 `json:"auth_timeout,omitempty"`
|
|
NoTLS bool `json:"no_tls,omitempty"`
|
|
TLSMap bool `json:"tls_map,omitempty"`
|
|
TLSPinnedCerts []string `json:"tls_pinned_certs,omitempty"`
|
|
SameOrigin bool `json:"same_origin,omitempty"`
|
|
AllowedOrigins []string `json:"allowed_origins,omitempty"`
|
|
Compression bool `json:"compression,omitempty"`
|
|
}
|
|
|
|
// VarzOptions are the options passed to Varz().
|
|
// Currently, there are no options defined.
|
|
type VarzOptions struct{}
|
|
|
|
func myUptime(d time.Duration) string {
|
|
// Just use total seconds for uptime, and display days / years
|
|
tsecs := d / time.Second
|
|
tmins := tsecs / 60
|
|
thrs := tmins / 60
|
|
tdays := thrs / 24
|
|
tyrs := tdays / 365
|
|
|
|
if tyrs > 0 {
|
|
return fmt.Sprintf("%dy%dd%dh%dm%ds", tyrs, tdays%365, thrs%24, tmins%60, tsecs%60)
|
|
}
|
|
if tdays > 0 {
|
|
return fmt.Sprintf("%dd%dh%dm%ds", tdays, thrs%24, tmins%60, tsecs%60)
|
|
}
|
|
if thrs > 0 {
|
|
return fmt.Sprintf("%dh%dm%ds", thrs, tmins%60, tsecs%60)
|
|
}
|
|
if tmins > 0 {
|
|
return fmt.Sprintf("%dm%ds", tmins, tsecs%60)
|
|
}
|
|
return fmt.Sprintf("%ds", tsecs)
|
|
}
|
|
|
|
// HandleRoot will show basic info and links to others handlers.
|
|
func (s *Server) HandleRoot(w http.ResponseWriter, r *http.Request) {
|
|
// This feels dumb to me, but is required: https://code.google.com/p/go/issues/detail?id=4799
|
|
if r.URL.Path != s.httpBasePath {
|
|
http.NotFound(w, r)
|
|
return
|
|
}
|
|
s.mu.Lock()
|
|
s.httpReqStats[RootPath]++
|
|
s.mu.Unlock()
|
|
|
|
// Calculate source url. If git set go directly to that tag, otherwise just main.
|
|
var srcUrl string
|
|
if gitCommit == _EMPTY_ {
|
|
srcUrl = "https://github.com/nats-io/nats-server"
|
|
} else {
|
|
srcUrl = fmt.Sprintf("https://github.com/nats-io/nats-server/tree/%s", gitCommit)
|
|
}
|
|
|
|
fmt.Fprintf(w, `<html lang="en">
|
|
<head>
|
|
<link rel="shortcut icon" href="https://nats.io/favicon.ico">
|
|
<style type="text/css">
|
|
body { font-family: ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans",sans-serif; font-size: 18; font-weight: light-bold; margin-left: 32px }
|
|
a { display:block; margin-left: 7px; padding-bottom: 6px; color: rgb(72 72 92); text-decoration: none }
|
|
a:hover { font-weight: 600; color: rgb(59 50 202) }
|
|
a.help { display:inline; font-weight: 600; color: rgb(59 50 202); font-size: 20}
|
|
a.last { padding-bottom: 16px }
|
|
a.version { font-size: 14; font-weight: 400; width: 312px; text-align: right; margin-top: -2rem }
|
|
a.version:hover { color: rgb(22 22 32) }
|
|
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<svg xmlns="http://www.w3.org/2000/svg" role="img" width="325" height="110" viewBox="-4.14 -3.89 436.28 119.03"><style>.st1{fill:#fff}.st2{fill:#34a574}</style><path fill="#27aae1" d="M4.3 84.6h42.2L70.7 107V84.6H103v-80H4.3v80zm15.9-61.3h18.5l35.6 33.2V23.3h11.8v42.9H68.2L32 32.4v33.8H20.2V23.3z"/><path d="M32 32.4l36.2 33.8h17.9V23.3H74.3v33.2L38.7 23.3H20.2v42.9H32z" class="st1"/><path d="M159.8 30.7L147 49h25.6z" class="st2"/><path d="M111.3 84.6H210v-80h-98.7v80zm41-61.5H168l30.8 43.2h-14.1l-5.8-8.3h-38.1l-5.8 8.3h-13.5l30.8-43.2z" class="st2"/><path d="M140.8 57.9h38.1l5.8 8.3h14.1L168 23.1h-15.7l-30.8 43.2H135l5.8-8.4zm19-27.2L172.6 49H147l12.8-18.3z" class="st1"/><path fill="#375c93" d="M218.3 84.6H317v-80h-98.7v80zm15.5-61.3h66.7V33h-27.2v33.2h-12.2V33h-27.3v-9.7z"/><path d="M261.1 66.2h12.2V33h27.2v-9.7h-66.7V33h27.3z" class="st1"/><path fill="#8dc63f" d="M325.3 4.6v80H424v-80h-98.7zm76.5 56.7c-3.2 3.2-10.2 5.7-26.8 5.7-12.3 0-24.1-1.9-30.7-4.7v-10c6.3 2.8 20.1 5.5 30.7 5.5 9.3 0 15.8-.3 17.5-2.1.6-.6.7-1.3.7-2 0-.8-.2-1.3-.7-1.8-1-1-2.6-1.7-17.4-2.1-15.7-.4-23.4-2-27-5.6-1.7-1.7-2.6-4.4-2.6-7.5 0-3.3.6-6.2 3.3-8.9 3.6-3.6 10.7-5.3 25.1-5.3 10.8 0 21.6 1.7 27.3 4v10.1c-6.5-2.8-17.8-4.8-27.2-4.8-10.4 0-14.8.6-16.2 2-.5.5-.8 1.1-.8 1.9 0 .9.2 1.5.7 2 1.3 1.3 6.1 1.7 17.3 1.9 16.4.4 23.5 1.8 27 5.2 1.8 1.8 2.8 4.7 2.8 7.7.1 3.2-.6 6.4-3 8.8z"/><path d="M375.2 39.5c-11.2-.2-16-.6-17.3-1.9-.5-.5-.7-1.1-.7-2 0-.8.3-1.4.8-1.9 1.3-1.3 5.8-2 16.2-2 9.4 0 20.7 2 27.2 4.8v-10c-5.7-2.3-16.6-4-27.3-4-14.5 0-21.6 1.8-25.1 5.3-2.7 2.7-3.3 5.6-3.3 8.9 0 3.1 1 5.8 2.6 7.5 3.6 3.6 11.3 5.2 27 5.6 14.8.4 16.4 1.1 17.4 2.1.5.5.7 1 .7 1.8 0 .7-.1 1.3-.7 2-1.8 1.8-8.3 2.1-17.5 2.1-10.6 0-24.3-2.6-30.7-5.5v10.1c6.6 2.8 18.4 4.7 30.7 4.7 16.6 0 23.6-2.5 26.8-5.7 2.4-2.4 3.1-5.6 3.1-8.9 0-3.1-1-5.9-2.8-7.7-3.6-3.5-10.7-4.9-27.1-5.3z" class="st1"/></svg>
|
|
|
|
<a href=%s class='version'>v%s</a>
|
|
|
|
</div>
|
|
<br/>
|
|
<a href=.%s>General</a>
|
|
<a href=.%s>JetStream</a>
|
|
<a href=.%s>Connections</a>
|
|
<a href=.%s>Accounts</a>
|
|
<a href=.%s>Subscriptions</a>
|
|
<a href=.%s>Routes</a>
|
|
<a href=.%s>LeafNodes</a>
|
|
<a href=.%s>Gateways</a>
|
|
<a href=.%s class=last>Health Probe</a>
|
|
<a href=https://docs.nats.io/running-a-nats-service/nats_admin/monitoring class="help">Help</a>
|
|
</body>
|
|
</html>`,
|
|
srcUrl,
|
|
VERSION,
|
|
s.basePath(VarzPath),
|
|
s.basePath(JszPath),
|
|
s.basePath(ConnzPath),
|
|
s.basePath(AccountzPath),
|
|
s.basePath(SubszPath),
|
|
s.basePath(RoutezPath),
|
|
s.basePath(LeafzPath),
|
|
s.basePath(GatewayzPath),
|
|
s.basePath(HealthzPath),
|
|
)
|
|
}
|
|
|
|
func (s *Server) updateJszVarz(js *jetStream, v *JetStreamVarz, doConfig bool) {
|
|
if doConfig {
|
|
js.mu.RLock()
|
|
// We want to snapshot the config since it will then be available outside
|
|
// of the js lock. So make a copy first, then point to this copy.
|
|
cfg := js.config
|
|
v.Config = &cfg
|
|
js.mu.RUnlock()
|
|
}
|
|
v.Stats = js.usageStats()
|
|
if mg := js.getMetaGroup(); mg != nil {
|
|
if ci := s.raftNodeToClusterInfo(mg); ci != nil {
|
|
v.Meta = &MetaClusterInfo{Name: ci.Name, Leader: ci.Leader, Size: mg.ClusterSize()}
|
|
if ci.Leader == s.info.Name {
|
|
v.Meta.Replicas = ci.Replicas
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Varz returns a Varz struct containing the server information.
|
|
func (s *Server) Varz(varzOpts *VarzOptions) (*Varz, error) {
|
|
var rss, vss int64
|
|
var pcpu float64
|
|
|
|
// We want to do that outside of the lock.
|
|
pse.ProcUsage(&pcpu, &rss, &vss)
|
|
|
|
s.mu.Lock()
|
|
js := s.js
|
|
// We need to create a new instance of Varz (with no reference
|
|
// whatsoever to anything stored in the server) since the user
|
|
// has access to the returned value.
|
|
v := s.createVarz(pcpu, rss)
|
|
s.mu.Unlock()
|
|
if js != nil {
|
|
s.updateJszVarz(js, &v.JetStream, true)
|
|
}
|
|
|
|
return v, nil
|
|
}
|
|
|
|
// Returns a Varz instance.
|
|
// Server lock is held on entry.
|
|
func (s *Server) createVarz(pcpu float64, rss int64) *Varz {
|
|
info := s.info
|
|
opts := s.getOpts()
|
|
c := &opts.Cluster
|
|
gw := &opts.Gateway
|
|
ln := &opts.LeafNode
|
|
mqtt := &opts.MQTT
|
|
ws := &opts.Websocket
|
|
clustTlsReq := c.TLSConfig != nil
|
|
gatewayTlsReq := gw.TLSConfig != nil
|
|
leafTlsReq := ln.TLSConfig != nil
|
|
leafTlsVerify := leafTlsReq && ln.TLSConfig.ClientAuth == tls.RequireAndVerifyClientCert
|
|
varz := &Varz{
|
|
ID: info.ID,
|
|
Version: info.Version,
|
|
Proto: info.Proto,
|
|
GitCommit: info.GitCommit,
|
|
GoVersion: info.GoVersion,
|
|
Name: info.Name,
|
|
Host: info.Host,
|
|
Port: info.Port,
|
|
IP: info.IP,
|
|
HTTPHost: opts.HTTPHost,
|
|
HTTPPort: opts.HTTPPort,
|
|
HTTPBasePath: opts.HTTPBasePath,
|
|
HTTPSPort: opts.HTTPSPort,
|
|
Cluster: ClusterOptsVarz{
|
|
Name: info.Cluster,
|
|
Host: c.Host,
|
|
Port: c.Port,
|
|
AuthTimeout: c.AuthTimeout,
|
|
TLSTimeout: c.TLSTimeout,
|
|
TLSRequired: clustTlsReq,
|
|
TLSVerify: clustTlsReq,
|
|
},
|
|
Gateway: GatewayOptsVarz{
|
|
Name: gw.Name,
|
|
Host: gw.Host,
|
|
Port: gw.Port,
|
|
AuthTimeout: gw.AuthTimeout,
|
|
TLSTimeout: gw.TLSTimeout,
|
|
TLSRequired: gatewayTlsReq,
|
|
TLSVerify: gatewayTlsReq,
|
|
Advertise: gw.Advertise,
|
|
ConnectRetries: gw.ConnectRetries,
|
|
Gateways: []RemoteGatewayOptsVarz{},
|
|
RejectUnknown: gw.RejectUnknown,
|
|
},
|
|
LeafNode: LeafNodeOptsVarz{
|
|
Host: ln.Host,
|
|
Port: ln.Port,
|
|
AuthTimeout: ln.AuthTimeout,
|
|
TLSTimeout: ln.TLSTimeout,
|
|
TLSRequired: leafTlsReq,
|
|
TLSVerify: leafTlsVerify,
|
|
Remotes: []RemoteLeafOptsVarz{},
|
|
},
|
|
MQTT: MQTTOptsVarz{
|
|
Host: mqtt.Host,
|
|
Port: mqtt.Port,
|
|
NoAuthUser: mqtt.NoAuthUser,
|
|
AuthTimeout: mqtt.AuthTimeout,
|
|
TLSMap: mqtt.TLSMap,
|
|
TLSTimeout: mqtt.TLSTimeout,
|
|
JsDomain: mqtt.JsDomain,
|
|
AckWait: mqtt.AckWait,
|
|
MaxAckPending: mqtt.MaxAckPending,
|
|
},
|
|
Websocket: WebsocketOptsVarz{
|
|
Host: ws.Host,
|
|
Port: ws.Port,
|
|
Advertise: ws.Advertise,
|
|
NoAuthUser: ws.NoAuthUser,
|
|
JWTCookie: ws.JWTCookie,
|
|
AuthTimeout: ws.AuthTimeout,
|
|
NoTLS: ws.NoTLS,
|
|
TLSMap: ws.TLSMap,
|
|
SameOrigin: ws.SameOrigin,
|
|
AllowedOrigins: copyStrings(ws.AllowedOrigins),
|
|
Compression: ws.Compression,
|
|
HandshakeTimeout: ws.HandshakeTimeout,
|
|
},
|
|
Start: s.start,
|
|
MaxSubs: opts.MaxSubs,
|
|
Cores: numCores,
|
|
MaxProcs: maxProcs,
|
|
Tags: opts.Tags,
|
|
TrustedOperatorsJwt: opts.operatorJWT,
|
|
TrustedOperatorsClaim: opts.TrustedOperators,
|
|
}
|
|
if len(opts.Routes) > 0 {
|
|
varz.Cluster.URLs = urlsToStrings(opts.Routes)
|
|
}
|
|
if l := len(gw.Gateways); l > 0 {
|
|
rgwa := make([]RemoteGatewayOptsVarz, l)
|
|
for i, r := range gw.Gateways {
|
|
rgwa[i] = RemoteGatewayOptsVarz{
|
|
Name: r.Name,
|
|
TLSTimeout: r.TLSTimeout,
|
|
}
|
|
}
|
|
varz.Gateway.Gateways = rgwa
|
|
}
|
|
if l := len(ln.Remotes); l > 0 {
|
|
rlna := make([]RemoteLeafOptsVarz, l)
|
|
for i, r := range ln.Remotes {
|
|
var deny *DenyRules
|
|
if len(r.DenyImports) > 0 || len(r.DenyExports) > 0 {
|
|
deny = &DenyRules{
|
|
Imports: r.DenyImports,
|
|
Exports: r.DenyExports,
|
|
}
|
|
}
|
|
rlna[i] = RemoteLeafOptsVarz{
|
|
LocalAccount: r.LocalAccount,
|
|
URLs: urlsToStrings(r.URLs),
|
|
TLSTimeout: r.TLSTimeout,
|
|
Deny: deny,
|
|
}
|
|
}
|
|
varz.LeafNode.Remotes = rlna
|
|
}
|
|
|
|
// Finish setting it up with fields that can be updated during
|
|
// configuration reload and runtime.
|
|
s.updateVarzConfigReloadableFields(varz)
|
|
s.updateVarzRuntimeFields(varz, true, pcpu, rss)
|
|
return varz
|
|
}
|
|
|
|
func urlsToStrings(urls []*url.URL) []string {
|
|
sURLs := make([]string, len(urls))
|
|
for i, u := range urls {
|
|
sURLs[i] = u.Host
|
|
}
|
|
return sURLs
|
|
}
|
|
|
|
// Invoked during configuration reload once options have possibly be changed
|
|
// and config load time has been set. If s.varz has not been initialized yet
|
|
// (because no pooling of /varz has been made), this function does nothing.
|
|
// Server lock is held on entry.
|
|
func (s *Server) updateVarzConfigReloadableFields(v *Varz) {
|
|
if v == nil {
|
|
return
|
|
}
|
|
opts := s.getOpts()
|
|
info := &s.info
|
|
v.AuthRequired = info.AuthRequired
|
|
v.TLSRequired = info.TLSRequired
|
|
v.TLSVerify = info.TLSVerify
|
|
v.MaxConn = opts.MaxConn
|
|
v.PingInterval = opts.PingInterval
|
|
v.MaxPingsOut = opts.MaxPingsOut
|
|
v.AuthTimeout = opts.AuthTimeout
|
|
v.MaxControlLine = opts.MaxControlLine
|
|
v.MaxPayload = int(opts.MaxPayload)
|
|
v.MaxPending = opts.MaxPending
|
|
v.TLSTimeout = opts.TLSTimeout
|
|
v.WriteDeadline = opts.WriteDeadline
|
|
v.ConfigLoadTime = s.configTime
|
|
// Update route URLs if applicable
|
|
if s.varzUpdateRouteURLs {
|
|
v.Cluster.URLs = urlsToStrings(opts.Routes)
|
|
s.varzUpdateRouteURLs = false
|
|
}
|
|
if s.sys != nil && s.sys.account != nil {
|
|
v.SystemAccount = s.sys.account.GetName()
|
|
}
|
|
v.MQTT.TLSPinnedCerts = getPinnedCertsAsSlice(opts.MQTT.TLSPinnedCerts)
|
|
v.Websocket.TLSPinnedCerts = getPinnedCertsAsSlice(opts.Websocket.TLSPinnedCerts)
|
|
}
|
|
|
|
func getPinnedCertsAsSlice(certs PinnedCertSet) []string {
|
|
if len(certs) == 0 {
|
|
return nil
|
|
}
|
|
res := make([]string, 0, len(certs))
|
|
for cn := range certs {
|
|
res = append(res, cn)
|
|
}
|
|
return res
|
|
}
|
|
|
|
// Updates the runtime Varz fields, that is, fields that change during
|
|
// runtime and that should be updated any time Varz() or polling of /varz
|
|
// is done.
|
|
// Server lock is held on entry.
|
|
func (s *Server) updateVarzRuntimeFields(v *Varz, forceUpdate bool, pcpu float64, rss int64) {
|
|
v.Now = time.Now().UTC()
|
|
v.Uptime = myUptime(time.Since(s.start))
|
|
v.Mem = rss
|
|
v.CPU = pcpu
|
|
if l := len(s.info.ClientConnectURLs); l > 0 {
|
|
v.ClientConnectURLs = append([]string(nil), s.info.ClientConnectURLs...)
|
|
}
|
|
if l := len(s.info.WSConnectURLs); l > 0 {
|
|
v.WSConnectURLs = append([]string(nil), s.info.WSConnectURLs...)
|
|
}
|
|
v.Connections = len(s.clients)
|
|
v.TotalConnections = s.totalClients
|
|
v.Routes = len(s.routes)
|
|
v.Remotes = len(s.remotes)
|
|
v.Leafs = len(s.leafs)
|
|
v.InMsgs = atomic.LoadInt64(&s.inMsgs)
|
|
v.InBytes = atomic.LoadInt64(&s.inBytes)
|
|
v.OutMsgs = atomic.LoadInt64(&s.outMsgs)
|
|
v.OutBytes = atomic.LoadInt64(&s.outBytes)
|
|
v.SlowConsumers = atomic.LoadInt64(&s.slowConsumers)
|
|
v.PinnedAccountFail = atomic.LoadUint64(&s.pinnedAccFail)
|
|
|
|
// Make sure to reset in case we are re-using.
|
|
v.Subscriptions = 0
|
|
s.accounts.Range(func(k, val interface{}) bool {
|
|
acc := val.(*Account)
|
|
v.Subscriptions += acc.sl.Count()
|
|
return true
|
|
})
|
|
|
|
v.HTTPReqStats = make(map[string]uint64, len(s.httpReqStats))
|
|
for key, val := range s.httpReqStats {
|
|
v.HTTPReqStats[key] = val
|
|
}
|
|
|
|
// Update Gateway remote urls if applicable
|
|
gw := s.gateway
|
|
gw.RLock()
|
|
if gw.enabled {
|
|
for i := 0; i < len(v.Gateway.Gateways); i++ {
|
|
g := &v.Gateway.Gateways[i]
|
|
rgw := gw.remotes[g.Name]
|
|
if rgw != nil {
|
|
rgw.RLock()
|
|
// forceUpdate is needed if user calls Varz() programmatically,
|
|
// since we need to create a new instance every time and the
|
|
// gateway's varzUpdateURLs may have been set to false after
|
|
// a web /varz inspection.
|
|
if forceUpdate || rgw.varzUpdateURLs {
|
|
// Make reuse of backend array
|
|
g.URLs = g.URLs[:0]
|
|
// rgw.urls is a map[string]*url.URL where the key is
|
|
// already in the right format (host:port, without any
|
|
// user info present).
|
|
for u := range rgw.urls {
|
|
g.URLs = append(g.URLs, u)
|
|
}
|
|
rgw.varzUpdateURLs = false
|
|
}
|
|
rgw.RUnlock()
|
|
} else if g.Name == gw.name && len(gw.ownCfgURLs) > 0 {
|
|
// This is a remote that correspond to this very same server.
|
|
// We report the URLs that were configured (if any).
|
|
// Since we don't support changes to the gateway configuration
|
|
// at this time, we could do this only if g.URLs has not been already
|
|
// set, but let's do it regardless in case we add support for
|
|
// gateway config reload.
|
|
g.URLs = g.URLs[:0]
|
|
g.URLs = append(g.URLs, gw.ownCfgURLs...)
|
|
}
|
|
}
|
|
}
|
|
gw.RUnlock()
|
|
}
|
|
|
|
// HandleVarz will process HTTP requests for server information.
|
|
func (s *Server) HandleVarz(w http.ResponseWriter, r *http.Request) {
|
|
var rss, vss int64
|
|
var pcpu float64
|
|
|
|
// We want to do that outside of the lock.
|
|
pse.ProcUsage(&pcpu, &rss, &vss)
|
|
|
|
// In response to http requests, we want to minimize mem copies
|
|
// so we use an object stored in the server. Creating/collecting
|
|
// server metrics is done under server lock, but we don't want
|
|
// to marshal under that lock. Still, we need to prevent concurrent
|
|
// http requests to /varz to update s.varz while marshal is
|
|
// happening, so we need a new lock that serialize those http
|
|
// requests and include marshaling.
|
|
s.varzMu.Lock()
|
|
|
|
// Use server lock to create/update the server's varz object.
|
|
s.mu.Lock()
|
|
var created bool
|
|
js := s.js
|
|
s.httpReqStats[VarzPath]++
|
|
if s.varz == nil {
|
|
s.varz = s.createVarz(pcpu, rss)
|
|
created = true
|
|
} else {
|
|
s.updateVarzRuntimeFields(s.varz, false, pcpu, rss)
|
|
}
|
|
s.mu.Unlock()
|
|
// Since locking is jetStream -> Server, need to update jetstream
|
|
// varz outside of server lock.
|
|
if js != nil {
|
|
var v JetStreamVarz
|
|
// Work on stack variable
|
|
s.updateJszVarz(js, &v, created)
|
|
// Now update server's varz
|
|
s.mu.Lock()
|
|
sv := &s.varz.JetStream
|
|
if created {
|
|
sv.Config = v.Config
|
|
}
|
|
sv.Stats = v.Stats
|
|
sv.Meta = v.Meta
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
// Do the marshaling outside of server lock, but under varzMu lock.
|
|
b, err := json.MarshalIndent(s.varz, "", " ")
|
|
s.varzMu.Unlock()
|
|
|
|
if err != nil {
|
|
s.Errorf("Error marshaling response to /varz request: %v", err)
|
|
}
|
|
|
|
// Handle response
|
|
ResponseHandler(w, r, b)
|
|
}
|
|
|
|
// GatewayzOptions are the options passed to Gatewayz()
|
|
type GatewayzOptions struct {
|
|
// Name will output only remote gateways with this name
|
|
Name string `json:"name"`
|
|
|
|
// Accounts indicates if accounts with its interest should be included in the results.
|
|
Accounts bool `json:"accounts"`
|
|
|
|
// AccountName will limit the list of accounts to that account name (makes Accounts implicit)
|
|
AccountName string `json:"account_name"`
|
|
}
|
|
|
|
// Gatewayz represents detailed information on Gateways
|
|
type Gatewayz struct {
|
|
ID string `json:"server_id"`
|
|
Now time.Time `json:"now"`
|
|
Name string `json:"name,omitempty"`
|
|
Host string `json:"host,omitempty"`
|
|
Port int `json:"port,omitempty"`
|
|
OutboundGateways map[string]*RemoteGatewayz `json:"outbound_gateways"`
|
|
InboundGateways map[string][]*RemoteGatewayz `json:"inbound_gateways"`
|
|
}
|
|
|
|
// RemoteGatewayz represents information about an outbound connection to a gateway
|
|
type RemoteGatewayz struct {
|
|
IsConfigured bool `json:"configured"`
|
|
Connection *ConnInfo `json:"connection,omitempty"`
|
|
Accounts []*AccountGatewayz `json:"accounts,omitempty"`
|
|
}
|
|
|
|
// AccountGatewayz represents interest mode for this account
|
|
type AccountGatewayz struct {
|
|
Name string `json:"name"`
|
|
InterestMode string `json:"interest_mode"`
|
|
NoInterestCount int `json:"no_interest_count,omitempty"`
|
|
InterestOnlyThreshold int `json:"interest_only_threshold,omitempty"`
|
|
TotalSubscriptions int `json:"num_subs,omitempty"`
|
|
NumQueueSubscriptions int `json:"num_queue_subs,omitempty"`
|
|
}
|
|
|
|
// Gatewayz returns a Gatewayz struct containing information about gateways.
|
|
func (s *Server) Gatewayz(opts *GatewayzOptions) (*Gatewayz, error) {
|
|
srvID := s.ID()
|
|
now := time.Now().UTC()
|
|
gw := s.gateway
|
|
gw.RLock()
|
|
if !gw.enabled {
|
|
gw.RUnlock()
|
|
gwz := &Gatewayz{
|
|
ID: srvID,
|
|
Now: now,
|
|
OutboundGateways: map[string]*RemoteGatewayz{},
|
|
InboundGateways: map[string][]*RemoteGatewayz{},
|
|
}
|
|
return gwz, nil
|
|
}
|
|
// Here gateways are enabled, so fill up more.
|
|
gwz := &Gatewayz{
|
|
ID: srvID,
|
|
Now: now,
|
|
Name: gw.name,
|
|
Host: gw.info.Host,
|
|
Port: gw.info.Port,
|
|
}
|
|
gw.RUnlock()
|
|
|
|
gwz.OutboundGateways = s.createOutboundsRemoteGatewayz(opts, now)
|
|
gwz.InboundGateways = s.createInboundsRemoteGatewayz(opts, now)
|
|
|
|
return gwz, nil
|
|
}
|
|
|
|
// Based on give options struct, returns if there is a filtered
|
|
// Gateway Name and if we should do report Accounts.
|
|
// Note that if Accounts is false but AccountName is not empty,
|
|
// then Accounts is implicitly set to true.
|
|
func getMonitorGWOptions(opts *GatewayzOptions) (string, bool) {
|
|
var name string
|
|
var accs bool
|
|
if opts != nil {
|
|
if opts.Name != _EMPTY_ {
|
|
name = opts.Name
|
|
}
|
|
accs = opts.Accounts
|
|
if !accs && opts.AccountName != _EMPTY_ {
|
|
accs = true
|
|
}
|
|
}
|
|
return name, accs
|
|
}
|
|
|
|
// Returns a map of gateways outbound connections.
|
|
// Based on options, will include a single or all gateways,
|
|
// with no/single/or all accounts interest information.
|
|
func (s *Server) createOutboundsRemoteGatewayz(opts *GatewayzOptions, now time.Time) map[string]*RemoteGatewayz {
|
|
targetGWName, doAccs := getMonitorGWOptions(opts)
|
|
|
|
if targetGWName != _EMPTY_ {
|
|
c := s.getOutboundGatewayConnection(targetGWName)
|
|
if c == nil {
|
|
return nil
|
|
}
|
|
outbounds := make(map[string]*RemoteGatewayz, 1)
|
|
_, rgw := createOutboundRemoteGatewayz(c, opts, now, doAccs)
|
|
outbounds[targetGWName] = rgw
|
|
return outbounds
|
|
}
|
|
|
|
var connsa [16]*client
|
|
var conns = connsa[:0]
|
|
|
|
s.getOutboundGatewayConnections(&conns)
|
|
|
|
outbounds := make(map[string]*RemoteGatewayz, len(conns))
|
|
for _, c := range conns {
|
|
name, rgw := createOutboundRemoteGatewayz(c, opts, now, doAccs)
|
|
if rgw != nil {
|
|
outbounds[name] = rgw
|
|
}
|
|
}
|
|
return outbounds
|
|
}
|
|
|
|
// Returns a RemoteGatewayz for a given outbound gw connection
|
|
func createOutboundRemoteGatewayz(c *client, opts *GatewayzOptions, now time.Time, doAccs bool) (string, *RemoteGatewayz) {
|
|
var name string
|
|
var rgw *RemoteGatewayz
|
|
|
|
c.mu.Lock()
|
|
if c.gw != nil {
|
|
rgw = &RemoteGatewayz{}
|
|
if doAccs {
|
|
rgw.Accounts = createOutboundAccountsGatewayz(opts, c.gw)
|
|
}
|
|
if c.gw.cfg != nil {
|
|
rgw.IsConfigured = !c.gw.cfg.isImplicit()
|
|
}
|
|
rgw.Connection = &ConnInfo{}
|
|
rgw.Connection.fill(c, c.nc, now)
|
|
name = c.gw.name
|
|
}
|
|
c.mu.Unlock()
|
|
|
|
return name, rgw
|
|
}
|
|
|
|
// Returns the list of accounts for this outbound gateway connection.
|
|
// Based on the options, it will be a single or all accounts for
|
|
// this outbound.
|
|
func createOutboundAccountsGatewayz(opts *GatewayzOptions, gw *gateway) []*AccountGatewayz {
|
|
if gw.outsim == nil {
|
|
return nil
|
|
}
|
|
|
|
var accName string
|
|
if opts != nil {
|
|
accName = opts.AccountName
|
|
}
|
|
if accName != _EMPTY_ {
|
|
ei, ok := gw.outsim.Load(accName)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
a := createAccountOutboundGatewayz(accName, ei)
|
|
return []*AccountGatewayz{a}
|
|
}
|
|
|
|
accs := make([]*AccountGatewayz, 0, 4)
|
|
gw.outsim.Range(func(k, v interface{}) bool {
|
|
name := k.(string)
|
|
a := createAccountOutboundGatewayz(name, v)
|
|
accs = append(accs, a)
|
|
return true
|
|
})
|
|
return accs
|
|
}
|
|
|
|
// Returns an AccountGatewayz for this gateway outbound connection
|
|
func createAccountOutboundGatewayz(name string, ei interface{}) *AccountGatewayz {
|
|
a := &AccountGatewayz{
|
|
Name: name,
|
|
InterestOnlyThreshold: gatewayMaxRUnsubBeforeSwitch,
|
|
}
|
|
if ei != nil {
|
|
e := ei.(*outsie)
|
|
e.RLock()
|
|
a.InterestMode = e.mode.String()
|
|
a.NoInterestCount = len(e.ni)
|
|
a.NumQueueSubscriptions = e.qsubs
|
|
a.TotalSubscriptions = int(e.sl.Count())
|
|
e.RUnlock()
|
|
} else {
|
|
a.InterestMode = Optimistic.String()
|
|
}
|
|
return a
|
|
}
|
|
|
|
// Returns a map of gateways inbound connections.
|
|
// Each entry is an array of RemoteGatewayz since a given server
|
|
// may have more than one inbound from the same remote gateway.
|
|
// Based on options, will include a single or all gateways,
|
|
// with no/single/or all accounts interest information.
|
|
func (s *Server) createInboundsRemoteGatewayz(opts *GatewayzOptions, now time.Time) map[string][]*RemoteGatewayz {
|
|
targetGWName, doAccs := getMonitorGWOptions(opts)
|
|
|
|
var connsa [16]*client
|
|
var conns = connsa[:0]
|
|
s.getInboundGatewayConnections(&conns)
|
|
|
|
m := make(map[string][]*RemoteGatewayz)
|
|
for _, c := range conns {
|
|
c.mu.Lock()
|
|
if c.gw != nil && (targetGWName == _EMPTY_ || targetGWName == c.gw.name) {
|
|
igws := m[c.gw.name]
|
|
if igws == nil {
|
|
igws = make([]*RemoteGatewayz, 0, 2)
|
|
}
|
|
rgw := &RemoteGatewayz{}
|
|
if doAccs {
|
|
rgw.Accounts = createInboundAccountsGatewayz(opts, c.gw)
|
|
}
|
|
rgw.Connection = &ConnInfo{}
|
|
rgw.Connection.fill(c, c.nc, now)
|
|
igws = append(igws, rgw)
|
|
m[c.gw.name] = igws
|
|
}
|
|
c.mu.Unlock()
|
|
}
|
|
return m
|
|
}
|
|
|
|
// Returns the list of accounts for this inbound gateway connection.
|
|
// Based on the options, it will be a single or all accounts for
|
|
// this inbound.
|
|
func createInboundAccountsGatewayz(opts *GatewayzOptions, gw *gateway) []*AccountGatewayz {
|
|
if gw.insim == nil {
|
|
return nil
|
|
}
|
|
|
|
var accName string
|
|
if opts != nil {
|
|
accName = opts.AccountName
|
|
}
|
|
if accName != _EMPTY_ {
|
|
e, ok := gw.insim[accName]
|
|
if !ok {
|
|
return nil
|
|
}
|
|
a := createInboundAccountGatewayz(accName, e)
|
|
return []*AccountGatewayz{a}
|
|
}
|
|
|
|
accs := make([]*AccountGatewayz, 0, 4)
|
|
for name, e := range gw.insim {
|
|
a := createInboundAccountGatewayz(name, e)
|
|
accs = append(accs, a)
|
|
}
|
|
return accs
|
|
}
|
|
|
|
// Returns an AccountGatewayz for this gateway inbound connection
|
|
func createInboundAccountGatewayz(name string, e *insie) *AccountGatewayz {
|
|
a := &AccountGatewayz{
|
|
Name: name,
|
|
InterestOnlyThreshold: gatewayMaxRUnsubBeforeSwitch,
|
|
}
|
|
if e != nil {
|
|
a.InterestMode = e.mode.String()
|
|
a.NoInterestCount = len(e.ni)
|
|
} else {
|
|
a.InterestMode = Optimistic.String()
|
|
}
|
|
return a
|
|
}
|
|
|
|
// HandleGatewayz process HTTP requests for route information.
|
|
func (s *Server) HandleGatewayz(w http.ResponseWriter, r *http.Request) {
|
|
s.mu.Lock()
|
|
s.httpReqStats[GatewayzPath]++
|
|
s.mu.Unlock()
|
|
|
|
accs, err := decodeBool(w, r, "accs")
|
|
if err != nil {
|
|
return
|
|
}
|
|
gwName := r.URL.Query().Get("gw_name")
|
|
accName := r.URL.Query().Get("acc_name")
|
|
if accName != _EMPTY_ {
|
|
accs = true
|
|
}
|
|
|
|
opts := &GatewayzOptions{
|
|
Name: gwName,
|
|
Accounts: accs,
|
|
AccountName: accName,
|
|
}
|
|
gw, err := s.Gatewayz(opts)
|
|
if err != nil {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
w.Write([]byte(err.Error()))
|
|
return
|
|
}
|
|
b, err := json.MarshalIndent(gw, "", " ")
|
|
if err != nil {
|
|
s.Errorf("Error marshaling response to /gatewayz request: %v", err)
|
|
}
|
|
|
|
// Handle response
|
|
ResponseHandler(w, r, b)
|
|
}
|
|
|
|
// Leafz represents detailed information on Leafnodes.
|
|
type Leafz struct {
|
|
ID string `json:"server_id"`
|
|
Now time.Time `json:"now"`
|
|
NumLeafs int `json:"leafnodes"`
|
|
Leafs []*LeafInfo `json:"leafs"`
|
|
}
|
|
|
|
// LeafzOptions are options passed to Leafz
|
|
type LeafzOptions struct {
|
|
// Subscriptions indicates that Leafz will return a leafnode's subscriptions
|
|
Subscriptions bool `json:"subscriptions"`
|
|
Account string `json:"account"`
|
|
}
|
|
|
|
// LeafInfo has detailed information on each remote leafnode connection.
|
|
type LeafInfo struct {
|
|
Account string `json:"account"`
|
|
IP string `json:"ip"`
|
|
Port int `json:"port"`
|
|
RTT string `json:"rtt,omitempty"`
|
|
InMsgs int64 `json:"in_msgs"`
|
|
OutMsgs int64 `json:"out_msgs"`
|
|
InBytes int64 `json:"in_bytes"`
|
|
OutBytes int64 `json:"out_bytes"`
|
|
NumSubs uint32 `json:"subscriptions"`
|
|
Subs []string `json:"subscriptions_list,omitempty"`
|
|
}
|
|
|
|
// Leafz returns a Leafz structure containing information about leafnodes.
|
|
func (s *Server) Leafz(opts *LeafzOptions) (*Leafz, error) {
|
|
// Grab leafnodes
|
|
var lconns []*client
|
|
s.mu.Lock()
|
|
if len(s.leafs) > 0 {
|
|
lconns = make([]*client, 0, len(s.leafs))
|
|
for _, ln := range s.leafs {
|
|
if opts != nil && opts.Account != "" {
|
|
ln.mu.Lock()
|
|
ok := ln.acc.Name == opts.Account
|
|
ln.mu.Unlock()
|
|
if !ok {
|
|
continue
|
|
}
|
|
}
|
|
lconns = append(lconns, ln)
|
|
}
|
|
}
|
|
s.mu.Unlock()
|
|
|
|
var leafnodes []*LeafInfo
|
|
if len(lconns) > 0 {
|
|
leafnodes = make([]*LeafInfo, 0, len(lconns))
|
|
for _, ln := range lconns {
|
|
ln.mu.Lock()
|
|
lni := &LeafInfo{
|
|
Account: ln.acc.Name,
|
|
IP: ln.host,
|
|
Port: int(ln.port),
|
|
RTT: ln.getRTT().String(),
|
|
InMsgs: atomic.LoadInt64(&ln.inMsgs),
|
|
OutMsgs: ln.outMsgs,
|
|
InBytes: atomic.LoadInt64(&ln.inBytes),
|
|
OutBytes: ln.outBytes,
|
|
NumSubs: uint32(len(ln.subs)),
|
|
}
|
|
if opts != nil && opts.Subscriptions {
|
|
lni.Subs = make([]string, 0, len(ln.subs))
|
|
for _, sub := range ln.subs {
|
|
lni.Subs = append(lni.Subs, string(sub.subject))
|
|
}
|
|
}
|
|
ln.mu.Unlock()
|
|
leafnodes = append(leafnodes, lni)
|
|
}
|
|
}
|
|
return &Leafz{
|
|
ID: s.ID(),
|
|
Now: time.Now().UTC(),
|
|
NumLeafs: len(leafnodes),
|
|
Leafs: leafnodes,
|
|
}, nil
|
|
}
|
|
|
|
// HandleLeafz process HTTP requests for leafnode information.
|
|
func (s *Server) HandleLeafz(w http.ResponseWriter, r *http.Request) {
|
|
s.mu.Lock()
|
|
s.httpReqStats[LeafzPath]++
|
|
s.mu.Unlock()
|
|
|
|
subs, err := decodeBool(w, r, "subs")
|
|
if err != nil {
|
|
return
|
|
}
|
|
l, err := s.Leafz(&LeafzOptions{subs, r.URL.Query().Get("acc")})
|
|
if err != nil {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
w.Write([]byte(err.Error()))
|
|
return
|
|
}
|
|
b, err := json.MarshalIndent(l, "", " ")
|
|
if err != nil {
|
|
s.Errorf("Error marshaling response to /leafz request: %v", err)
|
|
}
|
|
|
|
// Handle response
|
|
ResponseHandler(w, r, b)
|
|
}
|
|
|
|
// ResponseHandler handles responses for monitoring routes
|
|
func ResponseHandler(w http.ResponseWriter, r *http.Request, data []byte) {
|
|
// Get callback from request
|
|
callback := r.URL.Query().Get("callback")
|
|
// If callback is not empty then
|
|
if callback != "" {
|
|
// Response for JSONP
|
|
w.Header().Set("Content-Type", "application/javascript")
|
|
fmt.Fprintf(w, "%s(%s)", callback, data)
|
|
} else {
|
|
// Otherwise JSON
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Write(data)
|
|
}
|
|
}
|
|
|
|
func (reason ClosedState) String() string {
|
|
switch reason {
|
|
case ClientClosed:
|
|
return "Client Closed"
|
|
case AuthenticationTimeout:
|
|
return "Authentication Timeout"
|
|
case AuthenticationViolation:
|
|
return "Authentication Failure"
|
|
case TLSHandshakeError:
|
|
return "TLS Handshake Failure"
|
|
case SlowConsumerPendingBytes:
|
|
return "Slow Consumer (Pending Bytes)"
|
|
case SlowConsumerWriteDeadline:
|
|
return "Slow Consumer (Write Deadline)"
|
|
case WriteError:
|
|
return "Write Error"
|
|
case ReadError:
|
|
return "Read Error"
|
|
case ParseError:
|
|
return "Parse Error"
|
|
case StaleConnection:
|
|
return "Stale Connection"
|
|
case ProtocolViolation:
|
|
return "Protocol Violation"
|
|
case BadClientProtocolVersion:
|
|
return "Bad Client Protocol Version"
|
|
case WrongPort:
|
|
return "Incorrect Port"
|
|
case MaxConnectionsExceeded:
|
|
return "Maximum Connections Exceeded"
|
|
case MaxAccountConnectionsExceeded:
|
|
return "Maximum Account Connections Exceeded"
|
|
case MaxPayloadExceeded:
|
|
return "Maximum Message Payload Exceeded"
|
|
case MaxControlLineExceeded:
|
|
return "Maximum Control Line Exceeded"
|
|
case MaxSubscriptionsExceeded:
|
|
return "Maximum Subscriptions Exceeded"
|
|
case DuplicateRoute:
|
|
return "Duplicate Route"
|
|
case RouteRemoved:
|
|
return "Route Removed"
|
|
case ServerShutdown:
|
|
return "Server Shutdown"
|
|
case AuthenticationExpired:
|
|
return "Authentication Expired"
|
|
case WrongGateway:
|
|
return "Wrong Gateway"
|
|
case MissingAccount:
|
|
return "Missing Account"
|
|
case Revocation:
|
|
return "Credentials Revoked"
|
|
case InternalClient:
|
|
return "Internal Client"
|
|
case MsgHeaderViolation:
|
|
return "Message Header Violation"
|
|
case NoRespondersRequiresHeaders:
|
|
return "No Responders Requires Headers"
|
|
case ClusterNameConflict:
|
|
return "Cluster Name Conflict"
|
|
case DuplicateRemoteLeafnodeConnection:
|
|
return "Duplicate Remote LeafNode Connection"
|
|
case DuplicateClientID:
|
|
return "Duplicate Client ID"
|
|
case DuplicateServerName:
|
|
return "Duplicate Server Name"
|
|
case MinimumVersionRequired:
|
|
return "Minimum Version Required"
|
|
}
|
|
|
|
return "Unknown State"
|
|
}
|
|
|
|
// AccountzOptions are options passed to Accountz
|
|
type AccountzOptions struct {
|
|
// Account indicates that Accountz will return details for the account
|
|
Account string `json:"account"`
|
|
}
|
|
|
|
func newExtServiceLatency(l *serviceLatency) *jwt.ServiceLatency {
|
|
if l == nil {
|
|
return nil
|
|
}
|
|
return &jwt.ServiceLatency{
|
|
Sampling: jwt.SamplingRate(l.sampling),
|
|
Results: jwt.Subject(l.subject),
|
|
}
|
|
}
|
|
|
|
type ExtImport struct {
|
|
jwt.Import
|
|
Invalid bool `json:"invalid"`
|
|
Share bool `json:"share"`
|
|
Tracking bool `json:"tracking"`
|
|
TrackingHdr http.Header `json:"tracking_header,omitempty"`
|
|
Latency *jwt.ServiceLatency `json:"latency,omitempty"`
|
|
M1 *ServiceLatency `json:"m1,omitempty"`
|
|
}
|
|
|
|
type ExtExport struct {
|
|
jwt.Export
|
|
ApprovedAccounts []string `json:"approved_accounts,omitempty"`
|
|
RevokedAct map[string]time.Time `json:"revoked_activations,omitempty"`
|
|
}
|
|
|
|
type ExtVrIssues struct {
|
|
Description string `json:"description"`
|
|
Blocking bool `json:"blocking"`
|
|
Time bool `json:"time_check"`
|
|
}
|
|
|
|
type ExtMap map[string][]*MapDest
|
|
|
|
type AccountInfo struct {
|
|
AccountName string `json:"account_name"`
|
|
LastUpdate time.Time `json:"update_time,omitempty"`
|
|
IsSystem bool `json:"is_system,omitempty"`
|
|
Expired bool `json:"expired"`
|
|
Complete bool `json:"complete"`
|
|
JetStream bool `json:"jetstream_enabled"`
|
|
LeafCnt int `json:"leafnode_connections"`
|
|
ClientCnt int `json:"client_connections"`
|
|
SubCnt uint32 `json:"subscriptions"`
|
|
Mappings ExtMap `json:"mappings,omitempty"`
|
|
Exports []ExtExport `json:"exports,omitempty"`
|
|
Imports []ExtImport `json:"imports,omitempty"`
|
|
Jwt string `json:"jwt,omitempty"`
|
|
IssuerKey string `json:"issuer_key,omitempty"`
|
|
NameTag string `json:"name_tag,omitempty"`
|
|
Tags jwt.TagList `json:"tags,omitempty"`
|
|
Claim *jwt.AccountClaims `json:"decoded_jwt,omitempty"`
|
|
Vr []ExtVrIssues `json:"validation_result_jwt,omitempty"`
|
|
RevokedUser map[string]time.Time `json:"revoked_user,omitempty"`
|
|
Sublist *SublistStats `json:"sublist_stats,omitempty"`
|
|
Responses map[string]ExtImport `json:"responses,omitempty"`
|
|
}
|
|
|
|
type Accountz struct {
|
|
ID string `json:"server_id"`
|
|
Now time.Time `json:"now"`
|
|
SystemAccount string `json:"system_account,omitempty"`
|
|
Accounts []string `json:"accounts,omitempty"`
|
|
Account *AccountInfo `json:"account_detail,omitempty"`
|
|
}
|
|
|
|
// HandleAccountz process HTTP requests for account information.
|
|
func (s *Server) HandleAccountz(w http.ResponseWriter, r *http.Request) {
|
|
s.mu.Lock()
|
|
s.httpReqStats[AccountzPath]++
|
|
s.mu.Unlock()
|
|
if l, err := s.Accountz(&AccountzOptions{r.URL.Query().Get("acc")}); err != nil {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
w.Write([]byte(err.Error()))
|
|
} else if b, err := json.MarshalIndent(l, "", " "); err != nil {
|
|
s.Errorf("Error marshaling response to %s request: %v", AccountzPath, err)
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
w.Write([]byte(err.Error()))
|
|
} else {
|
|
ResponseHandler(w, r, b) // Handle response
|
|
}
|
|
}
|
|
|
|
func (s *Server) Accountz(optz *AccountzOptions) (*Accountz, error) {
|
|
a := &Accountz{
|
|
ID: s.ID(),
|
|
Now: time.Now().UTC(),
|
|
}
|
|
if sacc := s.SystemAccount(); sacc != nil {
|
|
a.SystemAccount = sacc.GetName()
|
|
}
|
|
if optz.Account == "" {
|
|
a.Accounts = []string{}
|
|
s.accounts.Range(func(key, value interface{}) bool {
|
|
a.Accounts = append(a.Accounts, key.(string))
|
|
return true
|
|
})
|
|
return a, nil
|
|
} else if aInfo, err := s.accountInfo(optz.Account); err != nil {
|
|
return nil, err
|
|
} else {
|
|
a.Account = aInfo
|
|
return a, nil
|
|
}
|
|
}
|
|
|
|
func newExtImport(v *serviceImport) ExtImport {
|
|
imp := ExtImport{
|
|
Invalid: true,
|
|
Import: jwt.Import{Type: jwt.Service},
|
|
}
|
|
if v != nil {
|
|
imp.Share = v.share
|
|
imp.Tracking = v.tracking
|
|
imp.Invalid = v.invalid
|
|
imp.Import = jwt.Import{
|
|
Subject: jwt.Subject(v.from),
|
|
Account: v.acc.Name,
|
|
Type: jwt.Service,
|
|
To: jwt.Subject(v.to),
|
|
}
|
|
imp.TrackingHdr = v.trackingHdr
|
|
imp.Latency = newExtServiceLatency(v.latency)
|
|
if v.m1 != nil {
|
|
m1 := *v.m1
|
|
imp.M1 = &m1
|
|
}
|
|
}
|
|
return imp
|
|
}
|
|
|
|
func (s *Server) accountInfo(accName string) (*AccountInfo, error) {
|
|
var a *Account
|
|
if v, ok := s.accounts.Load(accName); !ok {
|
|
return nil, fmt.Errorf("Account %s does not exist", accName)
|
|
} else {
|
|
a = v.(*Account)
|
|
}
|
|
isSys := a == s.SystemAccount()
|
|
a.mu.RLock()
|
|
defer a.mu.RUnlock()
|
|
var vrIssues []ExtVrIssues
|
|
claim, _ := jwt.DecodeAccountClaims(a.claimJWT) // ignore error
|
|
if claim != nil {
|
|
vr := jwt.ValidationResults{}
|
|
claim.Validate(&vr)
|
|
vrIssues = make([]ExtVrIssues, len(vr.Issues))
|
|
for i, v := range vr.Issues {
|
|
vrIssues[i] = ExtVrIssues{v.Description, v.Blocking, v.TimeCheck}
|
|
}
|
|
}
|
|
collectRevocations := func(revocations map[string]int64) map[string]time.Time {
|
|
l := len(revocations)
|
|
if l == 0 {
|
|
return nil
|
|
}
|
|
rev := make(map[string]time.Time, l)
|
|
for k, v := range revocations {
|
|
rev[k] = time.Unix(v, 0)
|
|
}
|
|
return rev
|
|
}
|
|
exports := []ExtExport{}
|
|
for k, v := range a.exports.services {
|
|
e := ExtExport{
|
|
Export: jwt.Export{
|
|
Subject: jwt.Subject(k),
|
|
Type: jwt.Service,
|
|
},
|
|
ApprovedAccounts: []string{},
|
|
}
|
|
if v != nil {
|
|
e.Latency = newExtServiceLatency(v.latency)
|
|
e.TokenReq = v.tokenReq
|
|
e.ResponseType = jwt.ResponseType(v.respType.String())
|
|
for name := range v.approved {
|
|
e.ApprovedAccounts = append(e.ApprovedAccounts, name)
|
|
}
|
|
e.RevokedAct = collectRevocations(v.actsRevoked)
|
|
}
|
|
exports = append(exports, e)
|
|
}
|
|
for k, v := range a.exports.streams {
|
|
e := ExtExport{
|
|
Export: jwt.Export{
|
|
Subject: jwt.Subject(k),
|
|
Type: jwt.Stream,
|
|
},
|
|
ApprovedAccounts: []string{},
|
|
}
|
|
if v != nil {
|
|
e.TokenReq = v.tokenReq
|
|
for name := range v.approved {
|
|
e.ApprovedAccounts = append(e.ApprovedAccounts, name)
|
|
}
|
|
e.RevokedAct = collectRevocations(v.actsRevoked)
|
|
}
|
|
exports = append(exports, e)
|
|
}
|
|
imports := []ExtImport{}
|
|
for _, v := range a.imports.streams {
|
|
imp := ExtImport{
|
|
Invalid: true,
|
|
Import: jwt.Import{Type: jwt.Stream},
|
|
}
|
|
if v != nil {
|
|
imp.Invalid = v.invalid
|
|
imp.Import = jwt.Import{
|
|
Subject: jwt.Subject(v.from),
|
|
Account: v.acc.Name,
|
|
Type: jwt.Stream,
|
|
LocalSubject: jwt.RenamingSubject(v.to),
|
|
}
|
|
}
|
|
imports = append(imports, imp)
|
|
}
|
|
for _, v := range a.imports.services {
|
|
imports = append(imports, newExtImport(v))
|
|
}
|
|
responses := map[string]ExtImport{}
|
|
for k, v := range a.exports.responses {
|
|
responses[k] = newExtImport(v)
|
|
}
|
|
mappings := ExtMap{}
|
|
for _, m := range a.mappings {
|
|
var dests []*MapDest
|
|
src := ""
|
|
if m == nil {
|
|
src = "nil"
|
|
if _, ok := mappings[src]; ok { // only set if not present (keep orig in case nil is used)
|
|
continue
|
|
}
|
|
dests = append(dests, &MapDest{})
|
|
} else {
|
|
src = m.src
|
|
for _, d := range m.dests {
|
|
dests = append(dests, &MapDest{d.tr.dest, d.weight, ""})
|
|
}
|
|
for c, cd := range m.cdests {
|
|
for _, d := range cd {
|
|
dests = append(dests, &MapDest{d.tr.dest, d.weight, c})
|
|
}
|
|
}
|
|
}
|
|
mappings[src] = dests
|
|
}
|
|
return &AccountInfo{
|
|
accName,
|
|
a.updated,
|
|
isSys,
|
|
a.expired,
|
|
!a.incomplete,
|
|
a.js != nil,
|
|
a.numLocalLeafNodes(),
|
|
a.numLocalConnections(),
|
|
a.sl.Count(),
|
|
mappings,
|
|
exports,
|
|
imports,
|
|
a.claimJWT,
|
|
a.Issuer,
|
|
a.nameTag,
|
|
a.tags,
|
|
claim,
|
|
vrIssues,
|
|
collectRevocations(a.usersRevoked),
|
|
a.sl.Stats(),
|
|
responses,
|
|
}, nil
|
|
}
|
|
|
|
// JSzOptions are options passed to Jsz
|
|
type JSzOptions struct {
|
|
Account string `json:"account,omitempty"`
|
|
Accounts bool `json:"accounts,omitempty"`
|
|
Streams bool `json:"streams,omitempty"`
|
|
Consumer bool `json:"consumer,omitempty"`
|
|
Config bool `json:"config,omitempty"`
|
|
LeaderOnly bool `json:"leader_only,omitempty"`
|
|
Offset int `json:"offset,omitempty"`
|
|
Limit int `json:"limit,omitempty"`
|
|
}
|
|
|
|
type StreamDetail struct {
|
|
Name string `json:"name"`
|
|
Cluster *ClusterInfo `json:"cluster,omitempty"`
|
|
Config *StreamConfig `json:"config,omitempty"`
|
|
State StreamState `json:"state,omitempty"`
|
|
Consumer []*ConsumerInfo `json:"consumer_detail,omitempty"`
|
|
Mirror *StreamSourceInfo `json:"mirror,omitempty"`
|
|
Sources []*StreamSourceInfo `json:"sources,omitempty"`
|
|
}
|
|
|
|
type AccountDetail struct {
|
|
Name string `json:"name"`
|
|
Id string `json:"id"`
|
|
JetStreamStats
|
|
Streams []StreamDetail `json:"stream_detail,omitempty"`
|
|
}
|
|
|
|
// MetaClusterInfo shows information about the meta group.
|
|
type MetaClusterInfo struct {
|
|
Name string `json:"name,omitempty"`
|
|
Leader string `json:"leader,omitempty"`
|
|
Replicas []*PeerInfo `json:"replicas,omitempty"`
|
|
Size int `json:"cluster_size"`
|
|
}
|
|
|
|
// JSInfo has detailed information on JetStream.
|
|
type JSInfo struct {
|
|
ID string `json:"server_id"`
|
|
Now time.Time `json:"now"`
|
|
Disabled bool `json:"disabled,omitempty"`
|
|
Config JetStreamConfig `json:"config,omitempty"`
|
|
JetStreamStats
|
|
Streams int `json:"streams"`
|
|
Consumers int `json:"consumers"`
|
|
Messages uint64 `json:"messages"`
|
|
Bytes uint64 `json:"bytes"`
|
|
Meta *MetaClusterInfo `json:"meta_cluster,omitempty"`
|
|
|
|
// aggregate raft info
|
|
AccountDetails []*AccountDetail `json:"account_details,omitempty"`
|
|
}
|
|
|
|
func (s *Server) accountDetail(jsa *jsAccount, optStreams, optConsumers, optCfg bool) *AccountDetail {
|
|
jsa.mu.RLock()
|
|
acc := jsa.account
|
|
name := acc.GetName()
|
|
id := name
|
|
if acc.nameTag != "" {
|
|
name = acc.nameTag
|
|
}
|
|
jsa.usageMu.RLock()
|
|
totalMem, totalStore := jsa.storageTotals()
|
|
detail := AccountDetail{
|
|
Name: name,
|
|
Id: id,
|
|
JetStreamStats: JetStreamStats{
|
|
Memory: totalMem,
|
|
Store: totalStore,
|
|
API: JetStreamAPIStats{
|
|
Total: jsa.apiTotal,
|
|
Errors: jsa.apiErrors,
|
|
},
|
|
},
|
|
Streams: make([]StreamDetail, 0, len(jsa.streams)),
|
|
}
|
|
jsa.usageMu.RUnlock()
|
|
var streams []*stream
|
|
if optStreams {
|
|
for _, stream := range jsa.streams {
|
|
streams = append(streams, stream)
|
|
}
|
|
}
|
|
jsa.mu.RUnlock()
|
|
|
|
if optStreams {
|
|
for _, stream := range streams {
|
|
ci := s.js.clusterInfo(stream.raftGroup())
|
|
var cfg *StreamConfig
|
|
if optCfg {
|
|
c := stream.config()
|
|
cfg = &c
|
|
}
|
|
sdet := StreamDetail{
|
|
Name: stream.name(),
|
|
State: stream.state(),
|
|
Cluster: ci,
|
|
Config: cfg,
|
|
Mirror: stream.mirrorInfo(),
|
|
Sources: stream.sourcesInfo(),
|
|
}
|
|
if optConsumers {
|
|
for _, consumer := range stream.getPublicConsumers() {
|
|
cInfo := consumer.info()
|
|
if cInfo == nil {
|
|
continue
|
|
}
|
|
|
|
if !optCfg {
|
|
cInfo.Config = nil
|
|
}
|
|
sdet.Consumer = append(sdet.Consumer, cInfo)
|
|
}
|
|
}
|
|
detail.Streams = append(detail.Streams, sdet)
|
|
}
|
|
}
|
|
return &detail
|
|
}
|
|
|
|
func (s *Server) JszAccount(opts *JSzOptions) (*AccountDetail, error) {
|
|
if s.js == nil {
|
|
return nil, fmt.Errorf("jetstream not enabled")
|
|
}
|
|
acc := opts.Account
|
|
account, ok := s.accounts.Load(acc)
|
|
if !ok {
|
|
return nil, fmt.Errorf("account %q not found", acc)
|
|
}
|
|
s.js.mu.RLock()
|
|
jsa, ok := s.js.accounts[account.(*Account).Name]
|
|
s.js.mu.RUnlock()
|
|
if !ok {
|
|
return nil, fmt.Errorf("account %q not jetstream enabled", acc)
|
|
}
|
|
return s.accountDetail(jsa, opts.Streams, opts.Consumer, opts.Config), nil
|
|
}
|
|
|
|
// helper to get cluster info from node via dummy group
|
|
func (s *Server) raftNodeToClusterInfo(node RaftNode) *ClusterInfo {
|
|
if node == nil {
|
|
return nil
|
|
}
|
|
peers := node.Peers()
|
|
peerList := make([]string, len(peers))
|
|
for i, p := range peers {
|
|
peerList[i] = p.ID
|
|
}
|
|
group := &raftGroup{
|
|
Name: _EMPTY_,
|
|
Peers: peerList,
|
|
node: node,
|
|
}
|
|
return s.js.clusterInfo(group)
|
|
}
|
|
|
|
// Jsz returns a Jsz structure containing information about JetStream.
|
|
func (s *Server) Jsz(opts *JSzOptions) (*JSInfo, error) {
|
|
// set option defaults
|
|
if opts == nil {
|
|
opts = &JSzOptions{}
|
|
}
|
|
if opts.Limit == 0 {
|
|
opts.Limit = 1024
|
|
}
|
|
if opts.Consumer {
|
|
opts.Streams = true
|
|
}
|
|
if opts.Streams {
|
|
opts.Accounts = true
|
|
}
|
|
|
|
jsi := &JSInfo{
|
|
ID: s.ID(),
|
|
Now: time.Now().UTC(),
|
|
}
|
|
|
|
js := s.getJetStream()
|
|
if js == nil || !js.isEnabled() {
|
|
if opts.LeaderOnly {
|
|
return nil, fmt.Errorf("%w: not leader", errSkipZreq)
|
|
}
|
|
|
|
jsi.Disabled = true
|
|
return jsi, nil
|
|
}
|
|
|
|
js.mu.RLock()
|
|
isLeader := js.cluster == nil || js.cluster.isLeader()
|
|
js.mu.RUnlock()
|
|
|
|
if opts.LeaderOnly && !isLeader {
|
|
return nil, fmt.Errorf("%w: not leader", errSkipZreq)
|
|
}
|
|
|
|
var accounts []*jsAccount
|
|
|
|
js.mu.RLock()
|
|
jsi.Config = js.config
|
|
for _, info := range js.accounts {
|
|
accounts = append(accounts, info)
|
|
}
|
|
js.mu.RUnlock()
|
|
|
|
if mg := js.getMetaGroup(); mg != nil {
|
|
if ci := s.raftNodeToClusterInfo(mg); ci != nil {
|
|
jsi.Meta = &MetaClusterInfo{Name: ci.Name, Leader: ci.Leader, Size: mg.ClusterSize()}
|
|
if isLeader {
|
|
jsi.Meta.Replicas = ci.Replicas
|
|
}
|
|
}
|
|
}
|
|
|
|
jsi.JetStreamStats = *js.usageStats()
|
|
|
|
filterIdx := -1
|
|
for i, jsa := range accounts {
|
|
if jsa.acc().GetName() == opts.Account {
|
|
filterIdx = i
|
|
}
|
|
jsa.mu.RLock()
|
|
streams := make([]*stream, 0, len(jsa.streams))
|
|
for _, stream := range jsa.streams {
|
|
streams = append(streams, stream)
|
|
}
|
|
jsa.mu.RUnlock()
|
|
jsi.Streams += len(streams)
|
|
for _, stream := range streams {
|
|
streamState := stream.state()
|
|
jsi.Messages += streamState.Msgs
|
|
jsi.Bytes += streamState.Bytes
|
|
jsi.Consumers += streamState.Consumers
|
|
}
|
|
}
|
|
|
|
// filter logic
|
|
if filterIdx != -1 {
|
|
accounts = []*jsAccount{accounts[filterIdx]}
|
|
} else if opts.Accounts {
|
|
if opts.Offset != 0 {
|
|
sort.Slice(accounts, func(i, j int) bool {
|
|
return strings.Compare(accounts[i].acc().Name, accounts[j].acc().Name) < 0
|
|
})
|
|
if opts.Offset > len(accounts) {
|
|
accounts = []*jsAccount{}
|
|
} else {
|
|
accounts = accounts[opts.Offset:]
|
|
}
|
|
}
|
|
if opts.Limit != 0 {
|
|
if opts.Limit < len(accounts) {
|
|
accounts = accounts[:opts.Limit]
|
|
}
|
|
}
|
|
} else {
|
|
accounts = []*jsAccount{}
|
|
}
|
|
if len(accounts) > 0 {
|
|
jsi.AccountDetails = make([]*AccountDetail, 0, len(accounts))
|
|
}
|
|
// if wanted, obtain accounts/streams/consumer
|
|
for _, jsa := range accounts {
|
|
detail := s.accountDetail(jsa, opts.Streams, opts.Consumer, opts.Config)
|
|
jsi.AccountDetails = append(jsi.AccountDetails, detail)
|
|
}
|
|
return jsi, nil
|
|
}
|
|
|
|
// HandleJsz process HTTP requests for jetstream information.
|
|
func (s *Server) HandleJsz(w http.ResponseWriter, r *http.Request) {
|
|
s.mu.Lock()
|
|
s.httpReqStats[JszPath]++
|
|
s.mu.Unlock()
|
|
accounts, err := decodeBool(w, r, "accounts")
|
|
if err != nil {
|
|
return
|
|
}
|
|
streams, err := decodeBool(w, r, "streams")
|
|
if err != nil {
|
|
return
|
|
}
|
|
consumers, err := decodeBool(w, r, "consumers")
|
|
if err != nil {
|
|
return
|
|
}
|
|
config, err := decodeBool(w, r, "config")
|
|
if err != nil {
|
|
return
|
|
}
|
|
offset, err := decodeInt(w, r, "offset")
|
|
if err != nil {
|
|
return
|
|
}
|
|
limit, err := decodeInt(w, r, "limit")
|
|
if err != nil {
|
|
return
|
|
}
|
|
leader, err := decodeBool(w, r, "leader-only")
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
l, err := s.Jsz(&JSzOptions{
|
|
r.URL.Query().Get("acc"),
|
|
accounts,
|
|
streams,
|
|
consumers,
|
|
config,
|
|
leader,
|
|
offset,
|
|
limit})
|
|
if err != nil {
|
|
w.WriteHeader(http.StatusBadRequest)
|
|
w.Write([]byte(err.Error()))
|
|
return
|
|
}
|
|
b, err := json.MarshalIndent(l, "", " ")
|
|
if err != nil {
|
|
s.Errorf("Error marshaling response to /jsz request: %v", err)
|
|
}
|
|
|
|
// Handle response
|
|
ResponseHandler(w, r, b)
|
|
}
|
|
|
|
type HealthStatus struct {
|
|
Status string `json:"status"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// https://tools.ietf.org/id/draft-inadarei-api-health-check-05.html
|
|
func (s *Server) HandleHealthz(w http.ResponseWriter, r *http.Request) {
|
|
s.mu.Lock()
|
|
s.httpReqStats[HealthzPath]++
|
|
s.mu.Unlock()
|
|
|
|
hs := s.healthz()
|
|
if hs.Error != _EMPTY_ {
|
|
s.Warnf("Healthcheck failed: %q", hs.Error)
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
}
|
|
b, err := json.Marshal(hs)
|
|
if err != nil {
|
|
s.Errorf("Error marshaling response to /healthz request: %v", err)
|
|
}
|
|
|
|
ResponseHandler(w, r, b)
|
|
}
|
|
|
|
// Generate health status.
|
|
func (s *Server) healthz() *HealthStatus {
|
|
var health = &HealthStatus{Status: "ok"}
|
|
|
|
if err := s.readyForConnections(time.Millisecond); err != nil {
|
|
health.Status = "error"
|
|
health.Error = err.Error()
|
|
return health
|
|
}
|
|
|
|
// Check JetStream
|
|
js := s.getJetStream()
|
|
if js == nil {
|
|
return health
|
|
}
|
|
|
|
// Clustered JetStream
|
|
js.mu.RLock()
|
|
defer js.mu.RUnlock()
|
|
|
|
cc := js.cluster
|
|
|
|
// Currently single server mode this is a no-op.
|
|
if cc == nil || cc.meta == nil {
|
|
return health
|
|
}
|
|
|
|
// If we are here we want to check for any assets assigned to us.
|
|
meta := cc.meta
|
|
ourID := meta.ID()
|
|
|
|
// If no meta leader.
|
|
if meta.GroupLeader() == _EMPTY_ {
|
|
health.Status = "unavailable"
|
|
health.Error = "JetStream has not established contact with a meta leader"
|
|
return health
|
|
}
|
|
// If we are not current with the meta leader.
|
|
if !meta.Current() {
|
|
health.Status = "unavailable"
|
|
health.Error = "JetStream is not current with the meta leader"
|
|
return health
|
|
}
|
|
|
|
// Range across all accounts, the streams assigned to them, and the consumers.
|
|
// If they are assigned to this server check their status.
|
|
for acc, asa := range cc.streams {
|
|
for stream, sa := range asa {
|
|
if sa.Group.isMember(ourID) {
|
|
// Make sure we can look up
|
|
if !cc.isStreamCurrent(acc, stream) {
|
|
health.Status = "unavailable"
|
|
health.Error = fmt.Sprintf("JetStream stream '%s > %s' is not current", acc, stream)
|
|
return health
|
|
}
|
|
// Now check consumers.
|
|
for consumer, ca := range sa.consumers {
|
|
if ca.Group.isMember(ourID) {
|
|
if !cc.isConsumerCurrent(acc, stream, consumer) {
|
|
health.Status = "unavailable"
|
|
health.Error = fmt.Sprintf("JetStream consumer '%s > %s > %s' is not current", acc, stream, consumer)
|
|
return health
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Success.
|
|
return health
|
|
}
|