// Copyright 2020 The NATS Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package server import ( "expvar" "fmt" "net" "reflect" "syscall" "unsafe" "golang.org/x/sys/unix" ) type TCPInfo = syscall.TCPInfo type TCPDiagnostics struct { Info TCPInfo UnreadData uint32 UnsentData uint32 } const PlatformCanGetSocketTCPInfo = true // GetSocketTCPDiagnostics relies upon a non-portable Linux-ism for returning a // lot of data about a connected socket. // We expect this to be called routinely, so we expect the caller to provide // the existing memory object to be written to. func GetSocketTCPDiagnostics(conn *net.TCPConn, diag *TCPDiagnostics) error { if conn == nil { return fmt.Errorf("GetSocketTCPDiagnostics: %w", ErrConnectionClosed) } rawConn, err := conn.SyscallConn() if err != nil { return fmt.Errorf("GetSocketTCPDiagnostics: %w", err) } *diag = TCPDiagnostics{} infoSize := syscall.SizeofTCPInfo err = nil rawConn.Control(func(fd uintptr) { sysErr := syscall.Errno(0) ret := uintptr(0) for sysErr == 0 || sysErr == syscall.EINTR { _, _, sysErr = syscall.Syscall6( syscall.SYS_GETSOCKOPT, fd, syscall.SOL_TCP, syscall.TCP_INFO, uintptr(unsafe.Pointer(&diag.Info)), uintptr(unsafe.Pointer(&infoSize)), 0) } if sysErr != 0 { err = fmt.Errorf("GetSocketTCPDiagnostics: getsockopt(TCP_INFO) failed: %w", sysErr) return } for sysErr == 0 || sysErr == syscall.EINTR { ret, _, sysErr = syscall.Syscall(syscall.SYS_IOCTL, fd, unix.SIOCINQ, 0) } if sysErr != 0 { err = fmt.Errorf("GetSocketTCPDiagnostics: getsockopt(SIOCINQ) failed: %w", sysErr) return } diag.UnreadData = uint32(ret) for sysErr == 0 || sysErr == syscall.EINTR { ret, _, sysErr = syscall.Syscall(syscall.SYS_IOCTL, fd, unix.SIOCOUTQ, 0) } if sysErr != 0 { err = fmt.Errorf("GetSocketTCPDiagnostics: getsockopt(SIOCOUTQ) failed: %w", sysErr) return } diag.UnsentData = uint32(ret) }) if err != nil { return err } return nil } type TCPInfoExpMetrics struct { // nb: the Golang syscall/ztypes_linux_amd64.go stops at Total_retrans // So we miss out on things like tcpi_notsent_bytes UnreadData expvar.Int UnsentData expvar.Int UnAckedPackets expvar.Int LostPackets expvar.Int RetransOutPackets expvar.Int TotalRetransPackets expvar.Int PathMTU expvar.Int LastDataSentMSec expvar.Int // result of jiffies_to_msecs(), now-timestamp LastDataRecvMSec expvar.Int // result of jiffies_to_msecs(), now-timestamp RTT expvar.Int RTTVariance expvar.Int } type TCPInfoExpMaps struct { UnreadData *expvar.Map UnsentData *expvar.Map UnAckedPackets *expvar.Map LostPackets *expvar.Map RetransOutPackets *expvar.Map TotalRetransPackets *expvar.Map PathMTU *expvar.Map LastDataSentMSec *expvar.Map LastDataRecvMSec *expvar.Map RTT *expvar.Map RTTVariance *expvar.Map } // Reflection note: we're using reflection once at startup, for maps // population, and once each time a new ID for a client is seen (eg, new // gateway), not on reconnect, so this should be rare enough that Reflection // should be better than repeating all those field-names yet another time. I // could possibly construct TCPInfoExpMaps dynamically but ... let's not go // down that hole. // NewTCPInfoExpMaps should only be called once in a given process. func NewTCPInfoExpMaps() *TCPInfoExpMaps { all := TCPInfoExpMaps{} t := reflect.TypeOf(&all).Elem() v := reflect.ValueOf(&all).Elem() for i := 0; i < t.NumField(); i++ { m := expvar.NewMap(t.Field(i).Name) v.Field(i).Set(reflect.ValueOf(m)) } return &all } func (m *TCPInfoExpMetrics) PopulateFromTCPDiagnostics(d *TCPDiagnostics, maps *TCPInfoExpMaps, fullLabel string) { // Might need to switch the label to be sanitized; we'll see. if v := maps.UnreadData.Get(fullLabel); v == nil { populateExpvarMapsTCPDiagnostics(maps, fullLabel, m) } m.UnreadData.Set(int64(d.UnreadData)) m.UnsentData.Set(int64(d.UnsentData)) m.UnAckedPackets.Set(int64(d.Info.Unacked)) m.LostPackets.Set(int64(d.Info.Lost)) m.RetransOutPackets.Set(int64(d.Info.Retrans)) m.TotalRetransPackets.Set(int64(d.Info.Total_retrans)) m.PathMTU.Set(int64(d.Info.Pmtu)) m.LastDataSentMSec.Set(int64(d.Info.Last_data_sent)) m.LastDataRecvMSec.Set(int64(d.Info.Last_data_recv)) m.RTT.Set(int64(d.Info.Rtt)) m.RTTVariance.Set(int64(d.Info.Rttvar)) } // Theoretically this could race against itself, but it would require two // callers for a given fullLabel, in different go-routines, so I'm skipping // that guard (to avoid complicating the reflect iteration with handling an // exception for a mutex, or moving the maps into a sub-struct). func populateExpvarMapsTCPDiagnostics(maps *TCPInfoExpMaps, fullLabel string, metrics *TCPInfoExpMetrics) { tm := reflect.TypeOf(maps).Elem() vm := reflect.ValueOf(maps).Elem() vmetrics := reflect.ValueOf(metrics) for i := 0; i < vm.NumField(); i++ { vm.Field(i).Interface().(*expvar.Map).Set(fullLabel, vmetrics.FieldByName(tm.Field(i).Name)) } } func (d *TCPDiagnostics) LogInteresting(log Logger, fullLabel string, previous *TCPInfoExpMetrics) { if int64(d.Info.Pmtu) != previous.PathMTU.Value() { log.Noticef("[SOCKET-DIAGNOSTICS] %q: Path MTU change: was %v now %v", fullLabel, previous.PathMTU.Value(), d.Info.Pmtu) } if int64(d.Info.Lost) != previous.LostPackets.Value() { v := previous.LostPackets.Value() log.Warnf("[SOCKET-DIAGNOSTICS] %q: LOST PACKETS: was %v now %v INCREASE: %v", fullLabel, v, int64(d.Info.Lost), v-int64(d.Info.Lost)) } if int64(d.Info.Total_retrans) != previous.TotalRetransPackets.Value() { v := previous.TotalRetransPackets.Value() log.Warnf("[SOCKET-DIAGNOSTICS] %q: total retransmission increase: was %v now %v INCREASE: %v", fullLabel, v, int64(d.Info.Total_retrans), v-int64(d.Info.Total_retrans)) } }