Track total connections, http request stats

2026-04-02 03:38:42 -07:00 · 2016-01-09 09:57:04 -08:00
parent b0c22e9dfd
commit 3af90e454f
5 changed files with 89 additions and 43 deletions
--- a/TODO.md
+++ b/TODO.md
@@ -14,11 +14,12 @@
 - [ ] Modify cluster support for single message across routes between pub/sub and d-queue
 - [ ] Memory limits/warnings?
 - [ ] Limit number of subscriptions a client can have, total memory usage etc.
- [ ] Gossip Protocol for discovery for clustering
 - [ ] Info updates contain other implicit route servers
 - [ ] Multi-tenant accounts with isolation of subject space
- [ ] Add to varz, time for slow consumers, peek or total connections, memory, etc.
- [ ] Add starttime and uptime to connz list.
+- [ ] Track last activity time per connection?
+- [X] Add total connections to varz so we won't miss spikes, etc.
+- [X] Add starttime and uptime to connz list.
+- [X] Gossip Protocol for discovery for clustering
 - [ ] Add in HTTP requests to varz?
 - [X] Add favico and help link for monitoring?
 - [X] Better user/pass support using bcrypt etc.
--- a/server/monitor.go
+++ b/server/monitor.go
@@ -71,6 +71,7 @@ func (s *Server) HandleConnz(w http.ResponseWriter, r *http.Request) {

 	// Walk the list
 	s.mu.Lock()
+	s.httpReqStats[ConnzPath]++
 	tlsRequired := s.info.TLSRequired
 	c.NumConns = len(s.clients)

@@ -213,6 +214,8 @@ func (s *Server) HandleRoutez(w http.ResponseWriter, r *http.Request) {

 	// Walk the list
 	s.mu.Lock()
+
+	s.httpReqStats[RoutezPath]++
 	rs.NumRoutes = len(s.routes)

 	for _, r := range s.routes {
@@ -252,6 +255,10 @@ func (s *Server) HandleRoutez(w http.ResponseWriter, r *http.Request) {

 // HandleStats process HTTP requests for subjects stats.
 func (s *Server) HandleSubsz(w http.ResponseWriter, r *http.Request) {
+	s.mu.Lock()
+	s.httpReqStats[SubszPath]++
+	s.mu.Unlock()
+
 	st := &Subsz{s.sl.Stats()}

 	b, err := json.MarshalIndent(st, "", "  ")
@@ -267,22 +274,25 @@ func (s *Server) HandleSubsz(w http.ResponseWriter, r *http.Request) {
 type Varz struct {
 	*Info
 	*Options
-	Port          int       `json:"port"`
-	MaxPayload    int       `json:"max_payload"`
-	Start         time.Time `json:"start"`
-	Now           time.Time `json:"now"`
-	Uptime        string    `json:"uptime"`
-	Mem           int64     `json:"mem"`
-	Cores         int       `json:"cores"`
-	CPU           float64   `json:"cpu"`
-	Connections   int       `json:"connections"`
-	Routes        int       `json:"routes"`
-	Remotes       int       `json:"remotes"`
-	InMsgs        int64     `json:"in_msgs"`
-	OutMsgs       int64     `json:"out_msgs"`
-	InBytes       int64     `json:"in_bytes"`
-	OutBytes      int64     `json:"out_bytes"`
-	SlowConsumers int64     `json:"slow_consumers"`
+	Port             int       `json:"port"`
+	MaxPayload       int       `json:"max_payload"`
+	Start            time.Time `json:"start"`
+	Now              time.Time `json:"now"`
+	Uptime           string    `json:"uptime"`
+	Mem              int64     `json:"mem"`
+	Cores            int       `json:"cores"`
+	CPU              float64   `json:"cpu"`
+	Connections      int       `json:"connections"`
+	TotalConnections uint64    `json:"total_connections"`
+	Routes           int       `json:"routes"`
+	Remotes          int       `json:"remotes"`
+	InMsgs           int64     `json:"in_msgs"`
+	OutMsgs          int64     `json:"out_msgs"`
+	InBytes          int64     `json:"in_bytes"`
+	OutBytes         int64     `json:"out_bytes"`
+	SlowConsumers    int64     `json:"slow_consumers"`
+
+	HTTPReqStats map[string]uint64 `json:"http_req_stats"`
 }

 type usage struct {
@@ -316,6 +326,14 @@ func myUptime(d time.Duration) string {

 // HandleRoot will show basic info and links to others handlers.
 func (s *Server) HandleRoot(w http.ResponseWriter, r *http.Request) {
+	// This feels dumb to me, but is required: https://code.google.com/p/go/issues/detail?id=4799
+	if r.URL.Path != "/" {
+		http.NotFound(w, r)
+		return
+	}
+	s.mu.Lock()
+	s.httpReqStats[RootPath]++
+	s.mu.Unlock()
 	fmt.Fprintf(w, rootHTML)
 }

@@ -330,6 +348,7 @@ func (s *Server) HandleVarz(w http.ResponseWriter, r *http.Request) {

 	s.mu.Lock()
 	v.Connections = len(s.clients)
+	v.TotalConnections = s.totalClients
 	v.Routes = len(s.routes)
 	v.Remotes = len(s.remotes)
 	v.InMsgs = s.inMsgs
@@ -337,6 +356,8 @@ func (s *Server) HandleVarz(w http.ResponseWriter, r *http.Request) {
 	v.OutMsgs = s.outMsgs
 	v.OutBytes = s.outBytes
 	v.SlowConsumers = s.slowConsumers
+	s.httpReqStats[VarzPath]++
+	v.HTTPReqStats = s.httpReqStats
 	s.mu.Unlock()

 	b, err := json.MarshalIndent(v, "", "  ")
--- a/server/monitor_test.go
+++ b/server/monitor_test.go
@@ -145,6 +145,9 @@ func TestVarz(t *testing.T) {
 	if v.Connections != 1 {
 		t.Fatalf("Expected Connections of 1, got %v\n", v.Connections)
 	}
+	if v.TotalConnections >= 1 {
+		t.Fatalf("Expected Total Connections of at least 1, got %v\n", v.TotalConnections)
+	}
 	if v.InMsgs != 1 {
 		t.Fatalf("Expected InMsgs of 1, got %v\n", v.InMsgs)
 	}
--- a/server/roothtml.go
+++ b/server/roothtml.go
@@ -6,7 +6,7 @@ package server
 var rootHTML = "" +
 `<html lang="en">
   <head>
-     <link href="data:image/x-icon;base64,AAABAAEAEBAAAAEAIABoBAAAFgAAACgAAAAQAAAAIAAAAAEAIAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+xY1CPsWNwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+xY1lPsWN/D7FjccAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+xY2MPsWN/z7Fjf8+xY3HAAAAAAAAAAAAAAAAAAAAAAAAAAA+xY3nPsWN5z7Fjec+xY3nPsWN5z7Fjec+xY3nPsWN/z7Fjf8+xY3/PsWN/z7Fjec+xY3nPsWN5z7Fjec+xY3nPsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/4vcuv/1/Pn/ld/A/z7Fjf8+xY3/PsWN/z7Fjf9KyZP/9fz5//X8+f/1/Pn/ctSr/z7Fjf8+xY3/PsWN/z7Fjf+V38D//////5/ixv8+xY3/PsWN/z7Fjf9Wy5n//////////////////////3nWr/8+xY3/PsWN/z7Fjf8+xY3/ld/A//////+f4sb/PsWN/z7Fjf9o0KP///////////+X3r//yO3d//////951q//PsWN/z7Fjf8+xY3/PsWN/5XfwP//////n+LG/z7Fjf991q////////////9/1rD/PsWN/8jt3f//////edav/z7Fjf8+xY3/PsWN/z7Fjf+V38D//////5/ixv+V3b3///////////9p0aT/PsWN/z7Fjf/I7d3//////3nWr/8+xY3/PsWN/z7Fjf8+xY3/ld/A//////////////////////9Wy5n/PsWN/z7Fjf8+xY3/yO3d//////951q//PsWN/z7Fjf8+xY3/PsWN/5XfwP////////////////9KyZP/PsWN/z7Fjf8+xY3/PsWN/8jt3f//////edav/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3/PsWN/z7Fjf8+xY3//98AAP+fAAD+HwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" rel="icon" type="image/x-icon"/>
+    <link rel="shortcut icon" href="http://nats.io/img/favicon.ico">
    <style type="text/css">
      body { font-family: “Century Gothic”, CenturyGothic, AppleGothic, sans-serif; font-size: 22; }
      a { margin-left: 32px; }
--- a/server/server.go
+++ b/server/server.go
@@ -44,23 +44,24 @@ type Server struct {
 	gcid uint64
 	grid uint64
 	stats
-	mu       sync.Mutex
-	info     Info
-	infoJSON []byte
-	sl       *sublist.Sublist
-	opts     *Options
-	auth     Auth
-	trace    bool
-	debug    bool
-	running  bool
-	listener net.Listener
-	clients  map[uint64]*client
-	routes   map[uint64]*client
-	remotes  map[string]*client
-	done     chan bool
-	start    time.Time
-	http     net.Listener
-
+	mu            sync.Mutex
+	info          Info
+	infoJSON      []byte
+	sl            *sublist.Sublist
+	opts          *Options
+	auth          Auth
+	trace         bool
+	debug         bool
+	running       bool
+	listener      net.Listener
+	clients       map[uint64]*client
+	routes        map[uint64]*client
+	remotes       map[string]*client
+	totalClients  uint64
+	done          chan bool
+	start         time.Time
+	http          net.Listener
+	httpReqStats  map[string]uint64
 	routeListener net.Listener
 	routeInfo     Info
 	rcQuit        chan bool
@@ -379,8 +380,27 @@ func (s *Server) StartHTTPSMonitoring() {
 	s.startMonitoring(true)
 }

+// HTTP endpoints
+const (
+	RootPath   = "/"
+	VarzPath   = "/varz"
+	ConnzPath  = "/connz"
+	RoutezPath = "/routez"
+	SubszPath  = "/subsz"
+)
+
 // Start the monitoring server
 func (s *Server) startMonitoring(secure bool) {
+
+	// Used to track HTTP requests
+	s.httpReqStats = map[string]uint64{
+		RootPath:   0,
+		VarzPath:   0,
+		ConnzPath:  0,
+		RoutezPath: 0,
+		SubszPath:  0,
+	}
+
 	var hp string
 	var err error

@@ -404,17 +424,17 @@ func (s *Server) startMonitoring(secure bool) {
 	mux := http.NewServeMux()

 	// Root
-	mux.HandleFunc("/", s.HandleRoot)
+	mux.HandleFunc(RootPath, s.HandleRoot)
 	// Varz
-	mux.HandleFunc("/varz", s.HandleVarz)
+	mux.HandleFunc(VarzPath, s.HandleVarz)
 	// Connz
-	mux.HandleFunc("/connz", s.HandleConnz)
+	mux.HandleFunc(ConnzPath, s.HandleConnz)
 	// Routez
-	mux.HandleFunc("/routez", s.HandleRoutez)
+	mux.HandleFunc(RoutezPath, s.HandleRoutez)
 	// Subz
+	mux.HandleFunc(SubszPath, s.HandleSubsz)
+	// Subz alias for backwards compatibility
 	mux.HandleFunc("/subscriptionsz", s.HandleSubsz)
-	// Subz
-	mux.HandleFunc("/subsz", s.HandleSubsz)

 	srv := &http.Server{
 		Addr:           hp,
@@ -439,6 +459,7 @@ func (s *Server) createClient(conn net.Conn) *client {
 	info := s.infoJSON
 	authRequired := s.info.AuthRequired
 	tlsRequired := s.info.TLSRequired
+	s.totalClients++
 	s.mu.Unlock()

 	// Grab lock