Add prof_block_rate option for enabling/configuring the block profile (#4587)

The new `prof_block_rate` configuration option allows the block profiler
to be enabled on demand after it was previously disabled in #4402. The
option is also reloadable so that it can be changed after startup.

Signed-off-by: Neil Twigg <neil@nats.io>

server/opts.go

@@ -307,6 +307,7 @@ type Options struct {
 	Websocket             WebsocketOpts     `json:"-"`
 	MQTT                  MQTTOpts          `json:"-"`
 	ProfPort              int               `json:"-"`
+	ProfBlockRate         int               `json:"-"`
 	PidFile               string            `json:"-"`
 	PortsFileDir          string            `json:"-"`
 	LogFile               string            `json:"-"`
@@ -1013,6 +1014,8 @@ func (o *Options) processConfigFileLine(k string, v interface{}, errors *[]error
 		o.PortsFileDir = v.(string)
 	case "prof_port":
 		o.ProfPort = int(v.(int64))
+	case "prof_block_rate":
+		o.ProfBlockRate = int(v.(int64))
 	case "max_control_line":
 		if v.(int64) > 1<<31-1 {
 			err := &configErr{tk, fmt.Sprintf("%s value is too big", k)}

server/reload.go

@@ -805,6 +805,16 @@ func (o *mqttInactiveThresholdReload) Apply(s *Server) {
 	s.Noticef("Reloaded: MQTT consumer_inactive_threshold = %v", o.newValue)
 }
 
+type profBlockRateReload struct {
+	noopOption
+	newValue int
+}
+
+func (o *profBlockRateReload) Apply(s *Server) {
+	s.setBlockProfileRate(o.newValue)
+	s.Noticef("Reloaded: block_prof_rate = %v", o.newValue)
+}
+
 type leafNodeOption struct {
 	noopOption
 	tlsFirstChanged    bool
@@ -1589,6 +1599,12 @@ func (s *Server) diffOptions(newOpts *Options) ([]option, error) {
 			diffOpts = append(diffOpts, &ocspOption{newValue: newValue.(*OCSPConfig)})
 		case "ocspcacheconfig":
 			diffOpts = append(diffOpts, &ocspResponseCacheOption{newValue: newValue.(*OCSPResponseCacheConfig)})
+		case "profblockrate":
+			new := newValue.(int)
+			old := oldValue.(int)
+			if new != old {
+				diffOpts = append(diffOpts, &profBlockRateReload{newValue: new})
+			}
 		default:
 			// TODO(ik): Implement String() on those options to have a nice print.
 			// %v is difficult to figure what's what, %+v print private fields and

server/server.go

@@ -2098,6 +2098,10 @@ func (s *Server) Start() {
 	// Pprof http endpoint for the profiler.
 	if opts.ProfPort != 0 {
 		s.StartProfiler()
+	} else {
+		// It's still possible to access this profile via a SYS endpoint, so set
+		// this anyway. (Otherwise StartProfiler would have called it.)
+		s.setBlockProfileRate(opts.ProfBlockRate)
 	}
 
 	if opts.ConfigFile != _EMPTY_ {
@@ -2701,6 +2705,8 @@ func (s *Server) StartProfiler() {
 	s.profiler = l
 	s.profilingServer = srv
 
+	s.setBlockProfileRate(opts.ProfBlockRate)
+
 	go func() {
 		// if this errors out, it's probably because the server is being shutdown
 		err := srv.Serve(l)
@@ -2718,6 +2724,15 @@ func (s *Server) StartProfiler() {
 	s.mu.Unlock()
 }
 
+func (s *Server) setBlockProfileRate(rate int) {
+	// Passing i ProfBlockRate <= 0 here will disable or > 0 will enable.
+	runtime.SetBlockProfileRate(rate)
+
+	if rate > 0 {
+		s.Warnf("Block profiling is enabled (rate %d), this may have a performance impact", rate)
+	}
+}
+
 // StartHTTPMonitoring will enable the HTTP monitoring port.
 // DEPRECATED: Should use StartMonitoring.
 func (s *Server) StartHTTPMonitoring() {