From 331f6ca3aafda8692cbeea8ec243d15374ffb9c4 Mon Sep 17 00:00:00 2001 From: Phil Pennock Date: Tue, 13 Oct 2020 19:31:10 -0400 Subject: [PATCH] WIP: theoretically, populate expvar maps --- server/client.go | 4 +++- server/server.go | 28 ++++++++++++---------- server/tcpinfo_linux.go | 53 ++++++++++++++++++++++++++++++++++++++++- server/tcpinfo_other.go | 8 ++++++- 4 files changed, 78 insertions(+), 15 deletions(-) diff --git a/server/client.go b/server/client.go index 8f592be1..9f09283a 100644 --- a/server/client.go +++ b/server/client.go @@ -4503,6 +4503,8 @@ func (c *client) diagnosticsLoop() { c.mu.Unlock() return } + expvarMaps := s.diagnosticExpvarMaps + routeID := c.route.remoteID c.mu.Unlock() var err error for { @@ -4519,7 +4521,7 @@ func (c *client) diagnosticsLoop() { } return } - (&c.diagMetrics).PopulateFromTCPDiagnostics(&c.diagTCPData) + (&c.diagMetrics).PopulateFromTCPDiagnostics(&c.diagTCPData, expvarMaps, routeID) c.mu.Unlock() } } diff --git a/server/server.go b/server/server.go index 2628c9d3..4041ffdb 100644 --- a/server/server.go +++ b/server/server.go @@ -224,6 +224,9 @@ type Server struct { // exporting account name the importer experienced issues with incompleteAccExporterMap sync.Map + + // Diagnostic metrics maps + diagnosticExpvarMaps *TCPInfoExpMaps } // Make sure all are 64bits for atomic use @@ -295,18 +298,19 @@ func NewServer(opts *Options) (*Server, error) { now := time.Now() s := &Server{ - kp: kp, - configFile: opts.ConfigFile, - info: info, - prand: rand.New(rand.NewSource(time.Now().UnixNano())), - opts: opts, - done: make(chan bool, 1), - start: now, - configTime: now, - gwLeafSubs: NewSublistWithCache(), - httpBasePath: httpBasePath, - eventIds: nuid.New(), - routesToSelf: make(map[string]struct{}), + kp: kp, + configFile: opts.ConfigFile, + info: info, + prand: rand.New(rand.NewSource(time.Now().UnixNano())), + opts: opts, + done: make(chan bool, 1), + start: now, + configTime: now, + gwLeafSubs: NewSublistWithCache(), + httpBasePath: httpBasePath, + eventIds: nuid.New(), + routesToSelf: make(map[string]struct{}), + diagnosticExpvarMaps: NewTCPInfoExpMaps(), } // Trusted root operator keys. diff --git a/server/tcpinfo_linux.go b/server/tcpinfo_linux.go index f3335e21..e21ac9c4 100644 --- a/server/tcpinfo_linux.go +++ b/server/tcpinfo_linux.go @@ -17,6 +17,7 @@ import ( "expvar" "fmt" "net" + "reflect" "syscall" "unsafe" @@ -106,7 +107,44 @@ type TCPInfoExpMetrics struct { RTTVariance expvar.Int } -func (m *TCPInfoExpMetrics) PopulateFromTCPDiagnostics(d *TCPDiagnostics) { +type TCPInfoExpMaps struct { + UnreadData *expvar.Map + UnsentData *expvar.Map + UnAckedPackets *expvar.Map + LostPackets *expvar.Map + RetransOutPackets *expvar.Map + TotalRetransPackets *expvar.Map + PathMTU *expvar.Map + LastDataSentMSec *expvar.Map + LastDataRecvMSec *expvar.Map + RTT *expvar.Map + RTTVariance *expvar.Map +} + +// Reflection note: we're using reflection once at startup, for maps +// population, and once each time a new ID for a client is seen (eg, new +// gateway), not on reconnect, so this should be rare enough that Reflection +// should be better than repeating all those field-names yet another time. I +// could possibly construct TCPInfoExpMaps dynamically but ... let's not go +// down that hole. + +// NewTCPInfoExpMaps should only be called once in a given process. +func NewTCPInfoExpMaps() *TCPInfoExpMaps { + all := TCPInfoExpMaps{} + t := reflect.TypeOf(&all).Elem() + v := reflect.ValueOf(&all).Elem() + for i := 0; i < t.NumField(); i++ { + m := expvar.NewMap(t.Field(i).Name) + v.Field(i).Set(reflect.ValueOf(m)) + } + return &all +} + +func (m *TCPInfoExpMetrics) PopulateFromTCPDiagnostics(d *TCPDiagnostics, maps *TCPInfoExpMaps, fullLabel string) { + // Might need to switch the label to be sanitized; we'll see. + if v := maps.UnreadData.Get(fullLabel); v == nil { + populateExpvarMapsTCPDiagnostics(maps, fullLabel, m) + } m.UnreadData.Set(int64(d.UnreadData)) m.UnsentData.Set(int64(d.UnsentData)) m.UnAckedPackets.Set(int64(d.Info.Unacked)) @@ -119,3 +157,16 @@ func (m *TCPInfoExpMetrics) PopulateFromTCPDiagnostics(d *TCPDiagnostics) { m.RTT.Set(int64(d.Info.Rtt)) m.RTTVariance.Set(int64(d.Info.Rttvar)) } + +// Theoretically this could race against itself, but it would require two +// callers for a given fullLabel, in different go-routines, so I'm skipping +// that guard (to avoid complicating the reflect iteration with handling an +// exception for a mutex, or moving the maps into a sub-struct). +func populateExpvarMapsTCPDiagnostics(maps *TCPInfoExpMaps, fullLabel string, metrics *TCPInfoExpMetrics) { + tm := reflect.TypeOf(maps).Elem() + vm := reflect.ValueOf(maps).Elem() + vmetrics := reflect.ValueOf(metrics) + for i := 0; i < vm.NumField(); i++ { + vm.Field(i).Interface().(*expvar.Map).Set(fullLabel, vmetrics.FieldByName(tm.Field(i).Name)) + } +} diff --git a/server/tcpinfo_other.go b/server/tcpinfo_other.go index a8132b33..837bf563 100644 --- a/server/tcpinfo_other.go +++ b/server/tcpinfo_other.go @@ -25,6 +25,7 @@ import ( type TCPInfo struct{} type TCPDiagnostics struct{} type TCPInfoExpMetrics struct{} +type TCPInfoExpMaps struct{} var ErrNotSupported = errors.New("error: operation not supported on this platform") @@ -37,6 +38,11 @@ func GetSocketTCPDiagnostics(conn *net.TCPConn, diag *TCPDiagnostics) error { return ErrNotImplemented } -func (m *TCPInfoExpMetrics) PopulateFromTCPDiagnostics(d *TCPDiagnostics) {} +func (m *TCPInfoExpMetrics) PopulateFromTCPDiagnostics(d *TCPDiagnostics, maps *TCPInfoExpMaps, fullLabel string) { +} + +func NewTCPInfoExpMaps() *TCPInfoExpMaps { + return &TCPInfoExpMaps{} +} // There will be other functions here, as we populate maps.