[ADDED] Support for route S2 compression

The new field `compression` in the `cluster{}` block allows to
specify which compression mode to use between servers.

It can be simply specified as a boolean or a string for the
simple modes, or as an object for the "s2_auto" mode where
a list of RTT thresholds can be specified.

By default, if no compression field is specified, the server
will use the s2_auto mode with default RTT thresholds of
10ms, 50ms and 100ms for the "uncompressed", "fast", "better"
and "best" modes.

```
cluster {
..
  # Possible values are "disabled", "off", "enabled", "on",
  # "accept", "s2_fast", "s2_better", "s2_best" or "s2_auto"
  compression: s2_fast
}
```

To specify a different list of thresholds for the s2_auto,
here is how it would look like:
```
cluster {
..
  compression: {
    mode: s2_auto
    # This means that for RTT up to 5ms (included), then
    # the compression level will be "uncompressed", then
    # from 5ms+ to 15ms, the mode will switch to "s2_fast",
    # then from 15ms+ to 50ms, the level will switch to
    # "s2_better", and anything above 50ms will result
    # in the "s2_best" compression mode.
    rtt_thresholds: [5ms, 15ms, 50ms]
  }
}
```

Note that the "accept" mode means that a server will accept
compression from a remote and switch to that same compression
mode, but will otherwise not initiate compression. That is,
if 2 servers are configured with "accept", then compression
will actually be "off". If one of the server had say s2_fast
then they would both use this mode.

If a server has compression mode set (other than "off") but
connects to an older server, there will be no compression between
those 2 routes.

Signed-off-by: Ivan Kozlovic <ivan@synadia.com>

server/reload_test.go

@@ -146,9 +146,10 @@ func TestConfigReloadUnsupported(t *testing.T) {
 		MaxPingsOut:    2,
 		WriteDeadline:  10 * time.Second,
 		Cluster: ClusterOpts{
-			Name: "abc",
-			Host: "127.0.0.1",
-			Port: -1,
+			Name:        "abc",
+			Host:        "127.0.0.1",
+			Port:        -1,
+			Compression: CompressionOpts{Mode: CompressionS2Auto, RTTThresholds: defaultCompressionS2AutoRTTThresholds},
 		},
 		NoSigs: true,
 	}
@@ -218,9 +219,10 @@ func TestConfigReloadInvalidConfig(t *testing.T) {
 		MaxPingsOut:    2,
 		WriteDeadline:  10 * time.Second,
 		Cluster: ClusterOpts{
-			Name: "abc",
-			Host: "127.0.0.1",
-			Port: -1,
+			Name:        "abc",
+			Host:        "127.0.0.1",
+			Port:        -1,
+			Compression: CompressionOpts{Mode: CompressionS2Auto, RTTThresholds: defaultCompressionS2AutoRTTThresholds},
 		},
 		NoSigs: true,
 	}
@@ -281,9 +283,10 @@ func TestConfigReload(t *testing.T) {
 		MaxPingsOut:    2,
 		WriteDeadline:  10 * time.Second,
 		Cluster: ClusterOpts{
-			Name: "abc",
-			Host: "127.0.0.1",
-			Port: server.ClusterAddr().Port,
+			Name:        "abc",
+			Host:        "127.0.0.1",
+			Port:        server.ClusterAddr().Port,
+			Compression: CompressionOpts{Mode: CompressionS2Auto, RTTThresholds: defaultCompressionS2AutoRTTThresholds},
 		},
 		NoSigs: true,
 	}
@@ -5361,3 +5364,177 @@ func TestConfigReloadGlobalAccountWithMappingAndJetStream(t *testing.T) {
 	_, err = js.StreamInfo("TEST")
 	require_NoError(t, err)
 }
+
+func TestConfigReloadRouteCompression(t *testing.T) {
+	org := testDefaultClusterCompression
+	testDefaultClusterCompression = _EMPTY_
+	defer func() { testDefaultClusterCompression = org }()
+
+	tmpl := `
+		port: -1
+		server_name: "%s"
+		cluster {
+			port: -1
+			name: "local"
+			%s
+			%s
+		}
+	`
+	conf1 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "A", _EMPTY_, "compression: accept")))
+	s1, o1 := RunServerWithConfig(conf1)
+	defer s1.Shutdown()
+
+	routes := fmt.Sprintf("routes: [\"nats://127.0.0.1:%d\"]", o1.Cluster.Port)
+	conf2 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "B", routes, "compression: accept")))
+	s2, _ := RunServerWithConfig(conf2)
+	defer s2.Shutdown()
+
+	// Run a 3rd server but make it as if it was an old server. We want to
+	// make sure that reload of s1 and s2 will not affect routes from s3 to
+	// s1/s2 because these do not support compression.
+	conf3 := createConfFile(t, []byte(fmt.Sprintf(tmpl, "C", routes, "compression: \"not supported\"")))
+	s3, _ := RunServerWithConfig(conf3)
+	defer s3.Shutdown()
+
+	checkClusterFormed(t, s1, s2, s3)
+
+	// Collect routes' cid from servers so we can check if routes are
+	// recreated when they should and are not when they should not.
+	collect := func(s *Server) map[uint64]struct{} {
+		m := make(map[uint64]struct{})
+		s.mu.RLock()
+		defer s.mu.RUnlock()
+		s.forEachRoute(func(r *client) {
+			r.mu.Lock()
+			m[r.cid] = struct{}{}
+			r.mu.Unlock()
+		})
+		return m
+	}
+	s1RouteIDs := collect(s1)
+	s2RouteIDs := collect(s2)
+	s3RouteIDs := collect(s3)
+	s3ID := s3.ID()
+
+	servers := []*Server{s1, s2}
+	checkCompMode := func(s1Expected, s2Expected string, shouldBeNew bool) {
+		t.Helper()
+		// We wait a bit to make sure that we have routes closed before
+		// checking that the cluster has (re)formed.
+		time.Sleep(100 * time.Millisecond)
+		// First, make sure that the cluster is formed
+		checkClusterFormed(t, s1, s2, s3)
+		// Then check that all routes are with the expected mode. We need to
+		// possibly wait a bit since there is negotiation going on.
+		checkFor(t, 2*time.Second, 50*time.Millisecond, func() error {
+			for _, s := range servers {
+				var err error
+				s.mu.RLock()
+				s.forEachRoute(func(r *client) {
+					if err != nil {
+						return
+					}
+					r.mu.Lock()
+					var exp string
+					var m map[uint64]struct{}
+					if r.route.remoteID == s3ID {
+						exp = CompressionNotSupported
+						m = s3RouteIDs
+					} else if s == s1 {
+						exp = s1Expected
+						m = s1RouteIDs
+					} else {
+						exp = s2Expected
+						m = s2RouteIDs
+					}
+					_, present := m[r.cid]
+					cm := r.route.compression
+					r.mu.Unlock()
+					if cm != exp {
+						err = fmt.Errorf("Expected route %v for server %s to have compression mode %q, got %q", r, s, exp, cm)
+					}
+					sbn := shouldBeNew
+					if exp == CompressionNotSupported {
+						// Override for routes to s3
+						sbn = false
+					}
+					if sbn && present {
+						err = fmt.Errorf("Expected route %v for server %s to be a new route, but it was already present", r, s)
+					} else if !sbn && !present {
+						err = fmt.Errorf("Expected route %v for server %s to not be new", r, s)
+					}
+				})
+				s.mu.RUnlock()
+				if err != nil {
+					return err
+				}
+			}
+			s1RouteIDs = collect(s1)
+			s2RouteIDs = collect(s2)
+			s3RouteIDs = collect(s3)
+			return nil
+		})
+	}
+	// Since both started with "accept", which means that a server can
+	// accept/switch to compression but not initiate compression, they
+	// should both be "off"
+	checkCompMode(CompressionOff, CompressionOff, false)
+
+	// Now reload s1 with "on" ("auto"), since s2 is *configured* with "accept",
+	// it won't use "auto" but instead fall back to "fast". S1 should be set
+	// to "uncompressed" because the default RTT threshold is 10ms and we should
+	// below that...
+	reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, "compression: on"))
+	checkCompMode(CompressionS2Uncompressed, CompressionS2Fast, true)
+	// Now reload s2
+	reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", routes, "compression: on"))
+	checkCompMode(CompressionS2Uncompressed, CompressionS2Uncompressed, false)
+
+	// Move on with "better"
+	reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, "compression: s2_better"))
+	// s1 should be at "better", but s2 still at "uncompressed"
+	checkCompMode(CompressionS2Better, CompressionS2Uncompressed, false)
+	// Now reload s2
+	reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", routes, "compression: s2_better"))
+	checkCompMode(CompressionS2Better, CompressionS2Better, false)
+
+	// Move to "best"
+	reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, "compression: s2_best"))
+	checkCompMode(CompressionS2Best, CompressionS2Better, false)
+	// Now reload s2
+	reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", routes, "compression: s2_best"))
+	checkCompMode(CompressionS2Best, CompressionS2Best, false)
+
+	// Now turn off
+	reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, "compression: off"))
+	checkCompMode(CompressionOff, CompressionOff, true)
+	// Now reload s2
+	reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", routes, "compression: off"))
+	checkCompMode(CompressionOff, CompressionOff, false)
+
+	// When "off" (and not "accept"), enabling 1 is not enough, the reload
+	// has to be done on both to take effect.
+	reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, "compression: s2_better"))
+	checkCompMode(CompressionOff, CompressionOff, true)
+	// Now reload s2
+	reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", routes, "compression: s2_better"))
+	checkCompMode(CompressionS2Better, CompressionS2Better, true)
+
+	// Try now to have different ones
+	reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, "compression: s2_best"))
+	// S1 should be "best" but S2 should have stayed at "better"
+	checkCompMode(CompressionS2Best, CompressionS2Better, false)
+
+	// Change compression setting back to "accept", which in that case we want
+	// to have a negotiation and use the remote's compression level. So
+	// connections should be re-created.
+	reloadUpdateConfig(t, s1, conf1, fmt.Sprintf(tmpl, "A", _EMPTY_, "compression: accept"))
+	checkCompMode(CompressionS2Better, CompressionS2Better, true)
+
+	// To avoid flapping, add a little sleep here to make sure we have things
+	// settled before reloading s2.
+	time.Sleep(100 * time.Millisecond)
+	// And if we do the same with s2, then we will end-up with no compression.
+	reloadUpdateConfig(t, s2, conf2, fmt.Sprintf(tmpl, "B", routes, "compression: accept"))
+	checkCompMode(CompressionOff, CompressionOff, true)
+}