From 0e2ab5eeea78f7744917c0ba85088ae578a57db9 Mon Sep 17 00:00:00 2001 From: Ivan Kozlovic Date: Mon, 25 Apr 2022 13:26:31 -0600 Subject: [PATCH] Changes to tests that run on Travis - Remove code coverage from Travis and add it to a GitHub Action that will be run as a nightly. - Use tag builds to exclude some tests, such as the "norace" or JS tests. Since "go test" does not support "negative" regexs, there is no other way. Signed-off-by: Ivan Kozlovic --- .github/workflows/cov.yaml | 43 + .travis.yml | 33 +- runTestsOnTravis.sh | 56 ++ scripts/cov.sh | 23 +- server/jetstream_cluster_test.go | 1343 +---------------------------- server/jetstream_helpers_test.go | 1329 ++++++++++++++++++++++++++++ server/jetstream_jwt_test.go | 763 ++++++++++++++++ server/jetstream_leafnode_test.go | 1208 ++++++++++++++++++++++++++ server/jetstream_test.go | 7 +- server/jwt_test.go | 731 ---------------- server/leafnode_test.go | 1180 ------------------------- server/norace_test.go | 4 +- test/fanout_test.go | 4 +- test/norace_test.go | 4 +- 14 files changed, 3457 insertions(+), 3271 deletions(-) create mode 100644 .github/workflows/cov.yaml create mode 100755 runTestsOnTravis.sh create mode 100644 server/jetstream_helpers_test.go create mode 100644 server/jetstream_jwt_test.go create mode 100644 server/jetstream_leafnode_test.go diff --git a/.github/workflows/cov.yaml b/.github/workflows/cov.yaml new file mode 100644 index 00000000..daf70532 --- /dev/null +++ b/.github/workflows/cov.yaml @@ -0,0 +1,43 @@ +name: NATS Server Code Coverage +on: + workflow_dispatch: {} + + schedule: + - cron: "40 4 * * *" + +jobs: + nightly_coverage: + runs-on: ubuntu-latest + + env: + GOPATH: /home/runner/work/nats-server + GO111MODULE: "on" + + steps: + - name: Checkout code + uses: actions/checkout@v1 + with: + path: src/github.com/nats-io/nats-server + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.17.x + + - name: Run code coverage + shell: bash --noprofile --norc -x -eo pipefail {0} + # Do not make the build fail even if code coverage reported + # a test failure. + run: | + ./scripts/cov.sh upload + + - name: Convert coverage.out to coverage.lcov + uses: jandelgado/gcov2lcov-action@v1.0.8 + with: + infile: acc.out + + - name: Coveralls + uses: coverallsapp/github-action@1.1.3 + with: + github-token: ${{ secrets.github_token }} + path-to-lcov: coverage.lcov diff --git a/.travis.yml b/.travis.yml index 8860dca9..8141561a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,31 +9,22 @@ language: go go: - 1.17.x - 1.16.x + addons: apt: packages: - rpm -env: -- GO111MODULE=on go_import_path: github.com/nats-io/nats-server -install: -- go install honnef.co/go/tools/cmd/staticcheck@v0.2.2 -- go install github.com/client9/misspell/cmd/misspell@latest -before_script: -- GO_LIST=$(go list ./...) -- go build -- $(exit $(go fmt $GO_LIST | wc -l)) -- go vet $GO_LIST -- find . -type f -name "*.go" | xargs misspell -error -locale US -- staticcheck $GO_LIST -script: -- set -e -- if [[ $TRAVIS_TAG ]]; then go test -v -run=TestVersionMatchesTag ./server; fi -- if [[ ! $TRAVIS_TAG ]]; then go test -v -run=TestNoRace --failfast -p=1 -timeout 30m ./...; fi -- if [[ ! $TRAVIS_TAG ]]; then if [[ "$TRAVIS_GO_VERSION" =~ 1.16 ]]; then ./scripts/cov.sh TRAVIS; else go test -v -race -p=1 --failfast -timeout 30m ./...; fi; fi -- set +e -after_success: -- if [[ ! $TRAVIS_TAG ]]; then if [[ "$TRAVIS_GO_VERSION" =~ 1.16 ]]; then $HOME/gopath/bin/goveralls -coverprofile=acc.out -service travis-ci; fi; fi + +env: +- TEST_SUITE=compile +- TEST_SUITE=no_race_tests +- TEST_SUITE=js_tests +- TEST_SUITE=js_cluster_tests +- TEST_SUITE=srv_pkg_non_js_tests +- TEST_SUITE=non_srv_pkg_tests + +script: ./runTestsOnTravis.sh $TEST_SUITE deploy: provider: script @@ -41,4 +32,4 @@ deploy: script: curl -sL http://git.io/goreleaser | bash on: tags: true - condition: $TRAVIS_GO_VERSION =~ 1.17 + condition: ($TRAVIS_GO_VERSION =~ 1.17) && ($TEST_SUITE = "compile") diff --git a/runTestsOnTravis.sh b/runTestsOnTravis.sh new file mode 100755 index 00000000..2d9a2c84 --- /dev/null +++ b/runTestsOnTravis.sh @@ -0,0 +1,56 @@ +#!/bin/sh + +set -e + +if [ "$1" = "compile" ]; then + + # We will compile and run some vet, spelling and some other checks. + + go install honnef.co/go/tools/cmd/staticcheck@v0.2.2; + go install github.com/client9/misspell/cmd/misspell@latest; + GO_LIST=$(go list ./...); + go build; + $(exit $(go fmt $GO_LIST | wc -l)); + go vet $GO_LIST; + find . -type f -name "*.go" | xargs misspell -error -locale US; + staticcheck $GO_LIST + if [ "$TRAVIS_TAG" != "" ]; then + go test -race -v -run=TestVersionMatchesTag ./server -count=1 -vet=off + fi + +elif [ "$1" = "no_race_tests" ]; then + + # Run tests without the `-race` flag. By convention, those tests start + # with `TestNoRace`. + + go test -v -p=1 -run=TestNoRace ./... -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "js_tests" ]; then + + # Run JetStream non-clustere tests. By convention, all JS tests start + # with `TestJetStream`. We exclude the clustered tests by using the + # `skip_js_cluster_tests` build tag. + + go test -race -v -run=TestJetStream ./server -tags=skip_js_cluster_tests -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "js_cluster_tests" ]; then + + # Run JetStream clustered tests. By convention, all JS clustered tests + # start with `TestJetStreamCluster`. + + go test -race -v -run=TestJetStreamCluster ./server -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "srv_pkg_non_js_tests" ]; then + + # Run all non JetStream tests in the server package. We exclude the + # JS tests by using the `skip_js_tests` build tag. + + go test -race -v -p=1 ./server/... -tags=skip_js_tests -count=1 -vet=off -timeout=30m -failfast + +elif [ "$1" = "non_srv_pkg_tests" ]; then + + # Run all tests of all non server package. + + go test -race -v -p=1 $(go list ./... | grep -v "/server") -count=1 -vet=off -timeout=30m -failfast + +fi diff --git a/scripts/cov.sh b/scripts/cov.sh index 393c5c84..558fc17f 100755 --- a/scripts/cov.sh +++ b/scripts/cov.sh @@ -1,23 +1,26 @@ -#!/bin/bash -e +#!/bin/bash # Run from directory above via ./scripts/cov.sh +# Do not set the -e flag because we don't a flapper to prevent the push to coverall. + export GO111MODULE="on" -go get github.com/mattn/goveralls -go get github.com/wadey/gocovmerge +go install github.com/wadey/gocovmerge@latest rm -rf ./cov mkdir cov -go test -v -failfast -covermode=atomic -coverprofile=./cov/conf.out -run='Test[^NoRace]' ./conf -timeout=20m -go test -v -failfast -covermode=atomic -coverprofile=./cov/log.out -run='Test[^NoRace]' ./logger -timeout=20m -go test -v -failfast -covermode=atomic -coverprofile=./cov/server.out -run='Test[^NoRace]' ./server -timeout=20m -go test -v -failfast -covermode=atomic -coverprofile=./cov/test.out -coverpkg=./server -run='Test[^NoRace]' ./test -timeout=20m +# Since it is difficult to get a full run without a flapper, do not use `-failfast`. +# It is better to have one flapper or two and still get the report than have +# to re-run the whole code coverage. One or two failed tests should not affect +# so much the code coverage. +go test -v -covermode=atomic -coverprofile=./cov/conf.out ./conf -timeout=20m -tags=skip_no_race_tests +go test -v -covermode=atomic -coverprofile=./cov/log.out ./logger -timeout=20m -tags=skip_no_race_tests +go test -v -covermode=atomic -coverprofile=./cov/server.out ./server -timeout=20m -tags=skip_no_race_tests +go test -v -covermode=atomic -coverprofile=./cov/test.out -coverpkg=./server ./test -timeout=20m -tags=skip_no_race_tests gocovmerge ./cov/*.out > acc.out rm -rf ./cov -# Without argument, launch browser results. We are going to push to coveralls only -# from Travis.yml and after success of the build (and result of pushing will not affect -# build result). +# If no argument passed, launch a browser to see the results. if [[ $1 == "" ]]; then go tool cover -html=acc.out fi diff --git a/server/jetstream_cluster_test.go b/server/jetstream_cluster_test.go index 6291b934..4a99ca22 100644 --- a/server/jetstream_cluster_test.go +++ b/server/jetstream_cluster_test.go @@ -11,6 +11,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !skip_js_tests && !skip_js_cluster_tests +// +build !skip_js_tests,!skip_js_cluster_tests + package server import ( @@ -21,7 +24,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "math/rand" "os" "path/filepath" @@ -82,7 +84,7 @@ func TestJetStreamClusterLeader(t *testing.T) { c.expectNoLeader() } -func TestJetStreamExpandCluster(t *testing.T) { +func TestJetStreamClusterExpand(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 2) defer c.shutdown() @@ -4576,7 +4578,7 @@ func TestJetStreamClusterSuperClusterMetaPlacement(t *testing.T) { } } -func TestJetStreamUniquePlacementTag(t *testing.T) { +func TestJetStreamClusterUniquePlacementTag(t *testing.T) { tmlp := ` listen: 127.0.0.1:-1 server_name: %s @@ -5287,7 +5289,7 @@ func TestJetStreamClusterMirrorAndSourcesFilteredConsumers(t *testing.T) { createConsumer("SCA", "baz") } -func TestJetStreamCrossAccountMirrorsAndSources(t *testing.T) { +func TestJetStreamClusterCrossAccountMirrorsAndSources(t *testing.T) { c := createJetStreamClusterWithTemplate(t, jsClusterMirrorSourceImportsTempl, "C1", 3) defer c.shutdown() @@ -5748,7 +5750,7 @@ func TestJetStreamClusterSuperClusterEphemeralCleanup(t *testing.T) { } } -func TestJetStreamSuperClusterConnectionCount(t *testing.T) { +func TestJetStreamClusterGatewayConnectionCount(t *testing.T) { sc := createJetStreamSuperClusterWithTemplate(t, jsClusterAccountsTempl, 3, 2) defer sc.shutdown() @@ -5820,7 +5822,7 @@ func TestJetStreamSuperClusterConnectionCount(t *testing.T) { }) } -func TestJetStreamSuperClusterDirectConsumersBrokenGateways(t *testing.T) { +func TestJetStreamClusterGatewayConsumersBrokenGateways(t *testing.T) { sc := createJetStreamSuperCluster(t, 1, 2) defer sc.shutdown() @@ -8706,7 +8708,7 @@ func TestJetStreamClusterConsumerLastActiveReporting(t *testing.T) { checkTimeDiff(ci.AckFloor.Last, nci.AckFloor.Last) } -func TestJetStreamRaceOnRAFTCreate(t *testing.T) { +func TestJetStreamClusterRaceOnRAFTCreate(t *testing.T) { c := createJetStreamClusterExplicit(t, "R3S", 3) defer c.shutdown() @@ -9095,7 +9097,7 @@ func TestJetStreamClusterMirrorAndSourceCrossNonNeighboringDomain(t *testing.T) }) } -func TestJetStreamSeal(t *testing.T) { +func TestJetStreamClusterSeal(t *testing.T) { s := RunBasicJetStreamServer() if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) @@ -9246,7 +9248,7 @@ func TestJetStreamClusteredStreamCreateIdempotent(t *testing.T) { addStream(t, nc, cfg) } -func TestJetStreamRollupsRequirePurge(t *testing.T) { +func TestJetStreamClusterRollupsRequirePurge(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() @@ -9276,7 +9278,7 @@ func TestJetStreamRollupsRequirePurge(t *testing.T) { } } -func TestJetStreamRollups(t *testing.T) { +func TestJetStreamClusterRollups(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() @@ -9352,7 +9354,7 @@ func TestJetStreamRollups(t *testing.T) { } } -func TestJetStreamRollupSubjectAndWatchers(t *testing.T) { +func TestJetStreamClusterRollupSubjectAndWatchers(t *testing.T) { c := createJetStreamClusterExplicit(t, "JSC", 3) defer c.shutdown() @@ -9705,7 +9707,7 @@ func TestJetStreamClusterAccountInfoForSystemAccount(t *testing.T) { } } -func TestJetStreamListFilter(t *testing.T) { +func TestJetStreamClusterListFilter(t *testing.T) { s := RunBasicJetStreamServer() if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) @@ -9761,7 +9763,7 @@ func TestJetStreamListFilter(t *testing.T) { t.Run("Clustered", func(t *testing.T) { testList(t, c.randomServer(), 3) }) } -func TestJetStreamConsumerUpdates(t *testing.T) { +func TestJetStreamClusterConsumerUpdates(t *testing.T) { s := RunBasicJetStreamServer() if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) @@ -9902,7 +9904,7 @@ func TestJetStreamConsumerUpdates(t *testing.T) { t.Run("Clustered", func(t *testing.T) { testConsumerUpdate(t, c.randomServer(), 2) }) } -func TestJetStreamSuperClusterPushConsumerInterest(t *testing.T) { +func TestJetStreamClusterGatewayPushConsumerInterest(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 2) defer sc.shutdown() @@ -10640,7 +10642,7 @@ func TestJetStreamClusterRedeliverBackoffs(t *testing.T) { } } -func TestJetStreamConsumerUpgrade(t *testing.T) { +func TestJetStreamClusterConsumerUpgrade(t *testing.T) { s := RunBasicJetStreamServer() if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) @@ -11171,7 +11173,7 @@ func TestJetStreamClusterMirrorOrSourceNotActiveReporting(t *testing.T) { } } -func TestJetStreamStreamAdvisories(t *testing.T) { +func TestJetStreamClusterStreamAdvisories(t *testing.T) { s := RunBasicJetStreamServer() if config := s.JetStreamConfig(); config != nil { defer removeDir(t, config.StoreDir) @@ -11326,7 +11328,7 @@ func TestJetStreamStreamAdvisories(t *testing.T) { t.Run("Clustered_R3", func(t *testing.T) { checkAdvisories(t, c.randomServer(), 3) }) } -func TestJetStreamRemovedPeersAndStreamsListAndDelete(t *testing.T) { +func TestJetStreamClusterRemovedPeersAndStreamsListAndDelete(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 3) defer sc.shutdown() @@ -11423,7 +11425,7 @@ func TestJetStreamRemovedPeersAndStreamsListAndDelete(t *testing.T) { require_Error(t, err, nats.ErrStreamNotFound) } -func TestJetStreamConsumerDeliverNewBug(t *testing.T) { +func TestJetStreamClusterConsumerDeliverNewBug(t *testing.T) { sc := createJetStreamSuperCluster(t, 3, 3) defer sc.shutdown() @@ -12482,7 +12484,7 @@ func TestJetStreamClusterImportConsumerStreamSubjectRemap(t *testing.T) { }) } -func TestJetStreamMaxHaAssets(t *testing.T) { +func TestJetStreamClusterMaxHaAssets(t *testing.T) { sc := createJetStreamSuperClusterWithTemplateAndModHook(t, ` listen: 127.0.0.1:-1 server_name: %s @@ -12726,1308 +12728,7 @@ func TestJetStreamClusterDeleteAndRestoreAndRestart(t *testing.T) { }) } -// Support functions - -// Used to setup superclusters for tests. -type supercluster struct { - t *testing.T - clusters []*cluster -} - -func (sc *supercluster) shutdown() { - if sc == nil { - return - } - for _, c := range sc.clusters { - shutdownCluster(c) - } -} - -func (sc *supercluster) randomServer() *Server { - return sc.randomCluster().randomServer() -} - -func (sc *supercluster) serverByName(sname string) *Server { - for _, c := range sc.clusters { - if s := c.serverByName(sname); s != nil { - return s - } - } - return nil -} - -func (sc *supercluster) waitOnStreamLeader(account, stream string) { - sc.t.Helper() - expires := time.Now().Add(30 * time.Second) - for time.Now().Before(expires) { - for _, c := range sc.clusters { - if leader := c.streamLeader(account, stream); leader != nil { - time.Sleep(200 * time.Millisecond) - return - } - } - time.Sleep(100 * time.Millisecond) - } - sc.t.Fatalf("Expected a stream leader for %q %q, got none", account, stream) -} - -var jsClusterAccountsTempl = ` - listen: 127.0.0.1:-1 - - server_name: %s - jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} - - leaf { - listen: 127.0.0.1:-1 - } - - cluster { - name: %s - listen: 127.0.0.1:%d - routes = [%s] - } - - no_auth_user: one - - accounts { - ONE { users = [ { user: "one", pass: "p" } ]; jetstream: enabled } - TWO { users = [ { user: "two", pass: "p" } ]; jetstream: enabled } - NOJS { users = [ { user: "nojs", pass: "p" } ] } - $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } - } -` - -var jsClusterTempl = ` - listen: 127.0.0.1:-1 - server_name: %s - jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} - - leaf { - listen: 127.0.0.1:-1 - } - - cluster { - name: %s - listen: 127.0.0.1:%d - routes = [%s] - } - - # For access to system account. - accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } -` - -var jsClusterMaxBytesTempl = ` - listen: 127.0.0.1:-1 - server_name: %s - jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} - - leaf { - listen: 127.0.0.1:-1 - } - - cluster { - name: %s - listen: 127.0.0.1:%d - routes = [%s] - } - - no_auth_user: u - - accounts { - $U { - users = [ { user: "u", pass: "p" } ] - jetstream: { - max_mem: 128MB - max_file: 18GB - max_bytes: true // Forces streams to indicate max_bytes. - } - } - $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } - } -` - -var jsClusterMaxBytesAccountLimitTempl = ` - listen: 127.0.0.1:-1 - server_name: %s - jetstream: {max_mem_store: 256MB, max_file_store: 4GB, store_dir: '%s'} - - leaf { - listen: 127.0.0.1:-1 - } - - cluster { - name: %s - listen: 127.0.0.1:%d - routes = [%s] - } - - no_auth_user: u - - accounts { - $U { - users = [ { user: "u", pass: "p" } ] - jetstream: { - max_mem: 128MB - max_file: 3GB - max_bytes: true // Forces streams to indicate max_bytes. - } - } - $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } - } -` - -var jsSuperClusterTempl = ` - %s - gateway { - name: %s - listen: 127.0.0.1:%d - gateways = [%s - ] - } - - system_account: "$SYS" -` - -var jsClusterLimitsTempl = ` - listen: 127.0.0.1:-1 - server_name: %s - jetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'} - - cluster { - name: %s - listen: 127.0.0.1:%d - routes = [%s] - } - - no_auth_user: u - - accounts { - ONE { - users = [ { user: "u", pass: "s3cr3t!" } ] - jetstream: enabled - } - $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } - } -` - -var jsMixedModeGlobalAccountTempl = ` - listen: 127.0.0.1:-1 - server_name: %s - jetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'} - - cluster { - name: %s - listen: 127.0.0.1:%d - routes = [%s] - } - - accounts {$SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } -` - -var jsGWTempl = `%s{name: %s, urls: [%s]}` - -func createJetStreamTaggedSuperCluster(t *testing.T) *supercluster { - sc := createJetStreamSuperCluster(t, 3, 3) - sc.waitOnPeerCount(9) - - reset := func(s *Server) { - s.mu.Lock() - rch := s.sys.resetCh - s.mu.Unlock() - if rch != nil { - rch <- struct{}{} - } - s.sendStatszUpdate() - } - - // Make first cluster AWS, US country code. - for _, s := range sc.clusterForName("C1").servers { - s.optsMu.Lock() - s.opts.Tags.Add("cloud:aws") - s.opts.Tags.Add("country:us") - s.optsMu.Unlock() - reset(s) - } - // Make second cluster GCP, UK country code. - for _, s := range sc.clusterForName("C2").servers { - s.optsMu.Lock() - s.opts.Tags.Add("cloud:gcp") - s.opts.Tags.Add("country:uk") - s.optsMu.Unlock() - reset(s) - } - // Make third cluster AZ, JP country code. - for _, s := range sc.clusterForName("C3").servers { - s.optsMu.Lock() - s.opts.Tags.Add("cloud:az") - s.opts.Tags.Add("country:jp") - s.optsMu.Unlock() - reset(s) - } - - ml := sc.leader() - js := ml.getJetStream() - require_True(t, js != nil) - js.mu.RLock() - defer js.mu.RUnlock() - cc := js.cluster - require_True(t, cc != nil) - - // Walk and make sure all tags are registered. - expires := time.Now().Add(10 * time.Second) - for time.Now().Before(expires) { - allOK := true - for _, p := range cc.meta.Peers() { - si, ok := ml.nodeToInfo.Load(p.ID) - require_True(t, ok) - ni := si.(nodeInfo) - if len(ni.tags) == 0 { - allOK = false - reset(sc.serverByName(ni.name)) - } - } - if allOK { - break - } - } - - return sc -} - -func createJetStreamSuperCluster(t *testing.T, numServersPer, numClusters int) *supercluster { - return createJetStreamSuperClusterWithTemplate(t, jsClusterTempl, numServersPer, numClusters) -} - -func createJetStreamSuperClusterWithTemplate(t *testing.T, tmpl string, numServersPer, numClusters int) *supercluster { - return createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, numServersPer, numClusters, nil) -} - -func createJetStreamSuperClusterWithTemplateAndModHook(t *testing.T, tmpl string, numServersPer, numClusters int, modify modifyCb) *supercluster { - t.Helper() - if numServersPer < 1 { - t.Fatalf("Number of servers must be >= 1") - } - if numClusters <= 1 { - t.Fatalf("Number of clusters must be > 1") - } - - startClusterPorts := []int{20_022, 22_022, 24_022} - startGatewayPorts := []int{20_122, 22_122, 24_122} - startClusterPort := startClusterPorts[rand.Intn(len(startClusterPorts))] - startGWPort := startGatewayPorts[rand.Intn(len(startGatewayPorts))] - - // Make the GWs form faster for the tests. - SetGatewaysSolicitDelay(10 * time.Millisecond) - defer ResetGatewaysSolicitDelay() - - cp, gp := startClusterPort, startGWPort - var clusters []*cluster - - var gws []string - // Build GWs first, will be same for all servers. - for i, port := 1, gp; i <= numClusters; i++ { - cn := fmt.Sprintf("C%d", i) - var urls []string - for n := 0; n < numServersPer; n++ { - urls = append(urls, fmt.Sprintf("nats-route://127.0.0.1:%d", port)) - port++ - } - gws = append(gws, fmt.Sprintf(jsGWTempl, "\n\t\t\t", cn, strings.Join(urls, ","))) - } - gwconf := strings.Join(gws, "") - - for i := 1; i <= numClusters; i++ { - cn := fmt.Sprintf("C%d", i) - // Go ahead and build configurations. - c := &cluster{servers: make([]*Server, 0, numServersPer), opts: make([]*Options, 0, numServersPer), name: cn} - - // Build out the routes that will be shared with all configs. - var routes []string - for port := cp; port < cp+numServersPer; port++ { - routes = append(routes, fmt.Sprintf("nats-route://127.0.0.1:%d", port)) - } - routeConfig := strings.Join(routes, ",") - - for si := 0; si < numServersPer; si++ { - storeDir := createDir(t, JetStreamStoreDir) - sn := fmt.Sprintf("%s-S%d", cn, si+1) - bconf := fmt.Sprintf(tmpl, sn, storeDir, cn, cp+si, routeConfig) - conf := fmt.Sprintf(jsSuperClusterTempl, bconf, cn, gp, gwconf) - gp++ - if modify != nil { - conf = modify(sn, cn, storeDir, conf) - } - s, o := RunServerWithConfig(createConfFile(t, []byte(conf))) - c.servers = append(c.servers, s) - c.opts = append(c.opts, o) - } - checkClusterFormed(t, c.servers...) - clusters = append(clusters, c) - cp += numServersPer - c.t = t - } - - // Wait for the supercluster to be formed. - egws := numClusters - 1 - for _, c := range clusters { - for _, s := range c.servers { - waitForOutboundGateways(t, s, egws, 2*time.Second) - } - } - - sc := &supercluster{t, clusters} - sc.waitOnLeader() - sc.waitOnAllCurrent() - - // Wait for all the peer nodes to be registered. - checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { - var peers []string - if ml := sc.leader(); ml != nil { - peers = ml.ActivePeers() - if len(peers) == numClusters*numServersPer { - return nil - } - } - return fmt.Errorf("Not correct number of peers, expected %d, got %d", numClusters*numServersPer, len(peers)) - }) - - if sc.leader() == nil { - sc.t.Fatalf("Expected a cluster leader, got none") - } - - return sc -} - -func (sc *supercluster) createLeafNodes(clusterName string, numServers int) *cluster { - // Create our leafnode cluster template first. - return sc.createLeafNodesWithDomain(clusterName, numServers, "") -} - -func (sc *supercluster) createLeafNodesWithDomain(clusterName string, numServers int, domain string) *cluster { - // Create our leafnode cluster template first. - return sc.randomCluster().createLeafNodes(clusterName, numServers, domain) -} - -func (sc *supercluster) createSingleLeafNode(extend bool) *Server { - return sc.randomCluster().createLeafNode(extend) -} - -func (sc *supercluster) leader() *Server { - for _, c := range sc.clusters { - if leader := c.leader(); leader != nil { - return leader - } - } - return nil -} - -func (sc *supercluster) waitOnLeader() { - sc.t.Helper() - expires := time.Now().Add(5 * time.Second) - for time.Now().Before(expires) { - for _, c := range sc.clusters { - if leader := c.leader(); leader != nil { - time.Sleep(250 * time.Millisecond) - return - } - } - time.Sleep(25 * time.Millisecond) - } - sc.t.Fatalf("Expected a cluster leader, got none") -} - -func (sc *supercluster) waitOnAllCurrent() { - for _, c := range sc.clusters { - c.waitOnAllCurrent() - } -} - -func (sc *supercluster) clusterForName(name string) *cluster { - for _, c := range sc.clusters { - if c.name == name { - return c - } - } - return nil -} - -func (sc *supercluster) randomCluster() *cluster { - clusters := append(sc.clusters[:0:0], sc.clusters...) - rand.Shuffle(len(clusters), func(i, j int) { clusters[i], clusters[j] = clusters[j], clusters[i] }) - return clusters[0] -} - -func (sc *supercluster) waitOnPeerCount(n int) { - sc.t.Helper() - sc.waitOnLeader() - leader := sc.leader() - expires := time.Now().Add(20 * time.Second) - for time.Now().Before(expires) { - peers := leader.JetStreamClusterPeers() - if len(peers) == n { - return - } - time.Sleep(100 * time.Millisecond) - } - sc.t.Fatalf("Expected a super cluster peer count of %d, got %d", n, len(leader.JetStreamClusterPeers())) -} - -var jsClusterMirrorSourceImportsTempl = ` - listen: 127.0.0.1:-1 - server_name: %s - jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} - - cluster { - name: %s - listen: 127.0.0.1:%d - routes = [%s] - } - - no_auth_user: dlc - - accounts { - JS { - jetstream: enabled - users = [ { user: "rip", pass: "pass" } ] - exports [ - { service: "$JS.API.CONSUMER.>" } # To create internal consumers to mirror/source. - { stream: "RI.DELIVER.SYNC.>" } # For the mirror/source consumers sending to IA via delivery subject. - { service: "$JS.FC.>" } - ] - } - IA { - jetstream: enabled - users = [ { user: "dlc", pass: "pass" } ] - imports [ - { service: { account: JS, subject: "$JS.API.CONSUMER.>"}, to: "RI.JS.API.CONSUMER.>" } - { stream: { account: JS, subject: "RI.DELIVER.SYNC.>"} } - { service: {account: JS, subject: "$JS.FC.>" }} - ] - } - $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } - } -` - -var jsClusterImportsTempl = ` - listen: 127.0.0.1:-1 - server_name: %s - jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} - - cluster { - name: %s - listen: 127.0.0.1:%d - routes = [%s] - } - - no_auth_user: dlc - - accounts { - JS { - jetstream: enabled - users = [ { user: "rip", pass: "pass" } ] - exports [ - { service: "$JS.API.>", response: stream } - { service: "TEST" } # For publishing to the stream. - { service: "$JS.ACK.TEST.*.>" } - ] - } - IA { - users = [ { user: "dlc", pass: "pass" } ] - imports [ - { service: { subject: "$JS.API.>", account: JS }} - { service: { subject: "TEST", account: JS }} - { service: { subject: "$JS.ACK.TEST.*.>", account: JS }} - ] - } - $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } - } -` - -func createMixedModeCluster(t *testing.T, tmpl string, clusterName, snPre string, numJsServers, numNonServers int, doJSConfig bool) *cluster { - t.Helper() - - if clusterName == _EMPTY_ || numJsServers < 0 || numNonServers < 1 { - t.Fatalf("Bad params") - } - - numServers := numJsServers + numNonServers - const startClusterPort = 23232 - - // Build out the routes that will be shared with all configs. - var routes []string - for cp := startClusterPort; cp < startClusterPort+numServers; cp++ { - routes = append(routes, fmt.Sprintf("nats-route://127.0.0.1:%d", cp)) - } - routeConfig := strings.Join(routes, ",") - - // Go ahead and build configurations and start servers. - c := &cluster{servers: make([]*Server, 0, numServers), opts: make([]*Options, 0, numServers), name: clusterName} - - for cp := startClusterPort; cp < startClusterPort+numServers; cp++ { - storeDir := createDir(t, JetStreamStoreDir) - - sn := fmt.Sprintf("%sS-%d", snPre, cp-startClusterPort+1) - conf := fmt.Sprintf(tmpl, sn, storeDir, clusterName, cp, routeConfig) - - // Disable JS here. - if cp-startClusterPort >= numJsServers { - // We can disable by commmenting it out, meaning no JS config, or can set the config up and just set disabled. - // e.g. jetstream: {domain: "SPOKE", enabled: false} - if doJSConfig { - conf = strings.Replace(conf, "jetstream: {", "jetstream: { enabled: false, ", 1) - } else { - conf = strings.Replace(conf, "jetstream: ", "# jetstream: ", 1) - } - } - - s, o := RunServerWithConfig(createConfFile(t, []byte(conf))) - c.servers = append(c.servers, s) - c.opts = append(c.opts, o) - } - c.t = t - - // Wait til we are formed and have a leader. - c.checkClusterFormed() - if numJsServers > 0 { - c.waitOnPeerCount(numJsServers) - } - - return c -} - -// This will create a cluster that is explicitly configured for the routes, etc. -// and also has a defined clustername. All configs for routes and cluster name will be the same. -func createJetStreamClusterExplicit(t *testing.T, clusterName string, numServers int) *cluster { - return createJetStreamClusterWithTemplate(t, jsClusterTempl, clusterName, numServers) -} - -func createJetStreamClusterWithTemplate(t *testing.T, tmpl string, clusterName string, numServers int) *cluster { - return createJetStreamClusterWithTemplateAndModHook(t, tmpl, clusterName, numServers, nil) -} - -func createJetStreamClusterWithTemplateAndModHook(t *testing.T, tmpl string, clusterName string, numServers int, modify modifyCb) *cluster { - startPorts := []int{7_022, 9_022, 11_022, 15_022} - port := startPorts[rand.Intn(len(startPorts))] - return createJetStreamClusterAndModHook(t, tmpl, clusterName, _EMPTY_, numServers, port, true, modify) -} - -func createJetStreamCluster(t *testing.T, tmpl string, clusterName, snPre string, numServers int, portStart int, waitOnReady bool) *cluster { - return createJetStreamClusterAndModHook(t, tmpl, clusterName, snPre, numServers, portStart, waitOnReady, nil) -} - -type modifyCb func(serverName, clusterName, storeDir, conf string) string - -func createJetStreamClusterAndModHook(t *testing.T, tmpl string, clusterName, snPre string, numServers int, portStart int, waitOnReady bool, modify modifyCb) *cluster { - t.Helper() - if clusterName == _EMPTY_ || numServers < 1 { - t.Fatalf("Bad params") - } - - // Flaky test prevention: - // Binding a socket to IP stack port 0 will bind an ephemeral port from an OS-specific range. - // If someone passes in to us a port spec which would cover that range, the test would be flaky. - // Adjust these ports to be the most inclusive across the port runner OSes. - // Linux: /proc/sys/net/ipv4/ip_local_port_range : 32768:60999 - // is useful, and shows there's no safe available range without OS-specific tuning. - // Our tests are usually run on Linux. Folks who care about other OSes: if you can't tune your test-runner OS to match, please - // propose a viable alternative. - const prohibitedPortFirst = 32768 - const prohibitedPortLast = 60999 - if (portStart >= prohibitedPortFirst && portStart <= prohibitedPortLast) || - (portStart+numServers-1 >= prohibitedPortFirst && portStart+numServers-1 <= prohibitedPortLast) { - t.Fatalf("test setup failure: may not specify a cluster port range which falls within %d:%d", prohibitedPortFirst, prohibitedPortLast) - } - - // Build out the routes that will be shared with all configs. - var routes []string - for cp := portStart; cp < portStart+numServers; cp++ { - routes = append(routes, fmt.Sprintf("nats-route://127.0.0.1:%d", cp)) - } - routeConfig := strings.Join(routes, ",") - - // Go ahead and build configurations and start servers. - c := &cluster{servers: make([]*Server, 0, numServers), opts: make([]*Options, 0, numServers), name: clusterName} - - for cp := portStart; cp < portStart+numServers; cp++ { - storeDir := createDir(t, JetStreamStoreDir) - sn := fmt.Sprintf("%sS-%d", snPre, cp-portStart+1) - conf := fmt.Sprintf(tmpl, sn, storeDir, clusterName, cp, routeConfig) - if modify != nil { - conf = modify(sn, clusterName, storeDir, conf) - } - s, o := RunServerWithConfig(createConfFile(t, []byte(conf))) - c.servers = append(c.servers, s) - c.opts = append(c.opts, o) - } - c.t = t - - // Wait til we are formed and have a leader. - c.checkClusterFormed() - if waitOnReady { - c.waitOnClusterReady() - } - - return c -} - -func (c *cluster) addInNewServer() *Server { - c.t.Helper() - sn := fmt.Sprintf("S-%d", len(c.servers)+1) - storeDir, _ := ioutil.TempDir(tempRoot, JetStreamStoreDir) - seedRoute := fmt.Sprintf("nats-route://127.0.0.1:%d", c.opts[0].Cluster.Port) - conf := fmt.Sprintf(jsClusterTempl, sn, storeDir, c.name, -1, seedRoute) - s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) - c.servers = append(c.servers, s) - c.opts = append(c.opts, o) - c.checkClusterFormed() - return s -} - -// This is tied to jsClusterAccountsTempl, so changes there to users needs to be reflected here. -func (c *cluster) createSingleLeafNodeNoSystemAccount() *Server { - as := c.randomServer() - lno := as.getOpts().LeafNode - ln1 := fmt.Sprintf("nats://one:p@%s:%d", lno.Host, lno.Port) - ln2 := fmt.Sprintf("nats://two:p@%s:%d", lno.Host, lno.Port) - conf := fmt.Sprintf(jsClusterSingleLeafNodeTempl, createDir(c.t, JetStreamStoreDir), ln1, ln2) - s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) - c.servers = append(c.servers, s) - c.opts = append(c.opts, o) - - checkLeafNodeConnectedCount(c.t, as, 2) - - return s -} - -// This is tied to jsClusterAccountsTempl, so changes there to users needs to be reflected here. -func (c *cluster) createSingleLeafNodeNoSystemAccountAndEnablesJetStream() *Server { - return c.createSingleLeafNodeNoSystemAccountAndEnablesJetStreamWithDomain(_EMPTY_, "nojs") -} - -func (c *cluster) createSingleLeafNodeNoSystemAccountAndEnablesJetStreamWithDomain(domain, user string) *Server { - tmpl := jsClusterSingleLeafNodeLikeNGSTempl - if domain != _EMPTY_ { - nsc := fmt.Sprintf("domain: %s, store_dir:", domain) - tmpl = strings.Replace(jsClusterSingleLeafNodeLikeNGSTempl, "store_dir:", nsc, 1) - } - as := c.randomServer() - lno := as.getOpts().LeafNode - ln := fmt.Sprintf("nats://%s:p@%s:%d", user, lno.Host, lno.Port) - conf := fmt.Sprintf(tmpl, createDir(c.t, JetStreamStoreDir), ln) - s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) - c.servers = append(c.servers, s) - c.opts = append(c.opts, o) - - checkLeafNodeConnectedCount(c.t, as, 1) - - return s -} - -var jsClusterSingleLeafNodeLikeNGSTempl = ` - listen: 127.0.0.1:-1 - server_name: LNJS - jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} - - leaf { remotes [ { urls: [ %s ] } ] } -` - -var jsClusterSingleLeafNodeTempl = ` - listen: 127.0.0.1:-1 - server_name: LNJS - jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} - - leaf { remotes [ - { urls: [ %s ], account: "JSY" } - { urls: [ %s ], account: "JSN" } ] - } - - accounts { - JSY { users = [ { user: "y", pass: "p" } ]; jetstream: true } - JSN { users = [ { user: "n", pass: "p" } ] } - $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } - } -` - -var jsClusterTemplWithLeafNode = ` - listen: 127.0.0.1:-1 - server_name: %s - jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} - - {{leaf}} - - cluster { - name: %s - listen: 127.0.0.1:%d - routes = [%s] - } - - # For access to system account. - accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } -` - -var jsClusterTemplWithLeafNodeNoJS = ` - listen: 127.0.0.1:-1 - server_name: %s - - # Need to keep below since it fills in the store dir by default so just comment out. - # jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} - - {{leaf}} - - cluster { - name: %s - listen: 127.0.0.1:%d - routes = [%s] - } - - # For access to system account. - accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } -` - -var jsClusterTemplWithSingleLeafNode = ` - listen: 127.0.0.1:-1 - server_name: %s - jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} - - {{leaf}} - - # For access to system account. - accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } -` - -var jsClusterTemplWithSingleLeafNodeNoJS = ` - listen: 127.0.0.1:-1 - server_name: %s - - # jetstream: {store_dir: '%s'} - - {{leaf}} - - # For access to system account. - accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } -` - -var jsLeafFrag = ` - leaf { - remotes [ - { urls: [ %s ] } - { urls: [ %s ], account: "$SYS" } - ] - } -` - -func (c *cluster) createLeafNodes(clusterName string, numServers int, domain string) *cluster { - return c.createLeafNodesWithStartPortAndDomain(clusterName, numServers, 22111, domain) -} - -func (c *cluster) createLeafNodesNoJS(clusterName string, numServers int) *cluster { - return c.createLeafNodesWithTemplateAndStartPort(jsClusterTemplWithLeafNodeNoJS, clusterName, numServers, 21333) -} - -func (c *cluster) createLeafNodesWithStartPortAndDomain(clusterName string, numServers int, portStart int, domain string) *cluster { - if domain == _EMPTY_ { - return c.createLeafNodesWithTemplateAndStartPort(jsClusterTemplWithLeafNode, clusterName, numServers, portStart) - } - tmpl := strings.Replace(jsClusterTemplWithLeafNode, "store_dir:", fmt.Sprintf(`domain: "%s", store_dir:`, domain), 1) - return c.createLeafNodesWithTemplateAndStartPort(tmpl, clusterName, numServers, portStart) -} - -func (c *cluster) createLeafNode(extend bool) *Server { - if extend { - return c.createLeafNodeWithTemplate("LNS", - strings.ReplaceAll(jsClusterTemplWithSingleLeafNode, "store_dir:", " extension_hint: will_extend, store_dir:")) - } else { - return c.createLeafNodeWithTemplate("LNS", jsClusterTemplWithSingleLeafNode) - } -} - -func (c *cluster) createLeafNodeWithTemplate(name, template string) *Server { - tmpl := c.createLeafSolicit(template) - conf := fmt.Sprintf(tmpl, name, createDir(c.t, JetStreamStoreDir)) - s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) - c.servers = append(c.servers, s) - c.opts = append(c.opts, o) - return s -} - -// Helper to generate the leaf solicit configs. -func (c *cluster) createLeafSolicit(tmpl string) string { - // Create our leafnode cluster template first. - var lns, lnss []string - for _, s := range c.servers { - if s.ClusterName() != c.name { - continue - } - ln := s.getOpts().LeafNode - lns = append(lns, fmt.Sprintf("nats://%s:%d", ln.Host, ln.Port)) - lnss = append(lnss, fmt.Sprintf("nats://admin:s3cr3t!@%s:%d", ln.Host, ln.Port)) - } - lnc := strings.Join(lns, ", ") - lnsc := strings.Join(lnss, ", ") - lconf := fmt.Sprintf(jsLeafFrag, lnc, lnsc) - return strings.Replace(tmpl, "{{leaf}}", lconf, 1) -} - -func (c *cluster) createLeafNodesWithTemplateMixedMode(template, clusterName string, numJsServers, numNonServers int, doJSConfig bool) *cluster { - // Create our leafnode cluster template first. - tmpl := c.createLeafSolicit(template) - pre := clusterName + "-" - - lc := createMixedModeCluster(c.t, tmpl, clusterName, pre, numJsServers, numNonServers, doJSConfig) - for _, s := range lc.servers { - checkLeafNodeConnectedCount(c.t, s, 2) - } - lc.waitOnClusterReadyWithNumPeers(numJsServers) - - return lc -} - -func (c *cluster) createLeafNodesWithTemplateAndStartPort(template, clusterName string, numServers int, portStart int) *cluster { - // Create our leafnode cluster template first. - tmpl := c.createLeafSolicit(template) - pre := clusterName + "-" - lc := createJetStreamCluster(c.t, tmpl, clusterName, pre, numServers, portStart, false) - for _, s := range lc.servers { - checkLeafNodeConnectedCount(c.t, s, 2) - } - return lc -} - -// Will add in the mapping for the account to each server. -func (c *cluster) addSubjectMapping(account, src, dest string) { - for _, s := range c.servers { - if s.ClusterName() != c.name { - continue - } - acc, err := s.LookupAccount(account) - if err != nil { - c.t.Fatalf("Unexpected error on %v: %v", s, err) - } - if err := acc.AddMapping(src, dest); err != nil { - c.t.Fatalf("Error adding mapping: %v", err) - } - } - // Make sure interest propagates. - time.Sleep(200 * time.Millisecond) -} - -// Adjust limits for the given account. -func (c *cluster) updateLimits(account string, newLimits map[string]JetStreamAccountLimits) { - c.t.Helper() - for _, s := range c.servers { - acc, err := s.LookupAccount(account) - if err != nil { - c.t.Fatalf("Unexpected error: %v", err) - } - if err := acc.UpdateJetStreamLimits(newLimits); err != nil { - c.t.Fatalf("Unexpected error: %v", err) - } - } -} - -// Hack for staticcheck -var skip = func(t *testing.T) { - t.SkipNow() -} - -func jsClientConnect(t *testing.T, s *Server, opts ...nats.Option) (*nats.Conn, nats.JetStreamContext) { - t.Helper() - nc, err := nats.Connect(s.ClientURL(), opts...) - if err != nil { - t.Fatalf("Failed to create client: %v", err) - } - js, err := nc.JetStream(nats.MaxWait(10 * time.Second)) - if err != nil { - t.Fatalf("Unexpected error getting JetStream context: %v", err) - } - return nc, js -} - -func jsClientConnectEx(t *testing.T, s *Server, domain string, opts ...nats.Option) (*nats.Conn, nats.JetStreamContext) { - t.Helper() - nc, err := nats.Connect(s.ClientURL(), opts...) - if err != nil { - t.Fatalf("Failed to create client: %v", err) - } - js, err := nc.JetStream(nats.MaxWait(10*time.Second), nats.Domain(domain)) - if err != nil { - t.Fatalf("Unexpected error getting JetStream context: %v", err) - } - return nc, js -} - -func checkSubsPending(t *testing.T, sub *nats.Subscription, numExpected int) { - t.Helper() - checkFor(t, 10*time.Second, 20*time.Millisecond, func() error { - if nmsgs, _, err := sub.Pending(); err != nil || nmsgs != numExpected { - return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, numExpected) - } - return nil - }) -} - -func fetchMsgs(t *testing.T, sub *nats.Subscription, numExpected int, totalWait time.Duration) []*nats.Msg { - t.Helper() - result := make([]*nats.Msg, 0, numExpected) - for start, count, wait := time.Now(), numExpected, totalWait; len(result) != numExpected; { - msgs, err := sub.Fetch(count, nats.MaxWait(wait)) - if err != nil { - t.Fatal(err) - } - result = append(result, msgs...) - count -= len(msgs) - if wait = totalWait - time.Since(start); wait < 0 { - break - } - } - if len(result) != numExpected { - t.Fatalf("Unexpected msg count, got %d, want %d", len(result), numExpected) - } - return result -} - -func (c *cluster) restartServer(rs *Server) *Server { - c.t.Helper() - index := -1 - var opts *Options - for i, s := range c.servers { - if s == rs { - index = i - break - } - } - if index < 0 { - c.t.Fatalf("Could not find server %v to restart", rs) - } - opts = c.opts[index] - s, o := RunServerWithConfig(opts.ConfigFile) - c.servers[index] = s - c.opts[index] = o - return s -} - -func (c *cluster) checkClusterFormed() { - c.t.Helper() - checkClusterFormed(c.t, c.servers...) -} - -func (c *cluster) waitOnPeerCount(n int) { - c.t.Helper() - c.waitOnLeader() - leader := c.leader() - for leader == nil { - c.waitOnLeader() - leader = c.leader() - } - expires := time.Now().Add(30 * time.Second) - for time.Now().Before(expires) { - if peers := leader.JetStreamClusterPeers(); len(peers) == n { - return - } - time.Sleep(100 * time.Millisecond) - leader = c.leader() - for leader == nil { - c.waitOnLeader() - leader = c.leader() - } - } - c.t.Fatalf("Expected a cluster peer count of %d, got %d", n, len(leader.JetStreamClusterPeers())) -} - -func (c *cluster) waitOnConsumerLeader(account, stream, consumer string) { - c.t.Helper() - expires := time.Now().Add(20 * time.Second) - for time.Now().Before(expires) { - if leader := c.consumerLeader(account, stream, consumer); leader != nil { - time.Sleep(200 * time.Millisecond) - return - } - time.Sleep(100 * time.Millisecond) - } - c.t.Fatalf("Expected a consumer leader for %q %q %q, got none", account, stream, consumer) -} - -func (c *cluster) consumerLeader(account, stream, consumer string) *Server { - c.t.Helper() - for _, s := range c.servers { - if s.JetStreamIsConsumerLeader(account, stream, consumer) { - return s - } - } - return nil -} - -func (c *cluster) randomNonConsumerLeader(account, stream, consumer string) *Server { - c.t.Helper() - for _, s := range c.servers { - if !s.JetStreamIsConsumerLeader(account, stream, consumer) { - return s - } - } - return nil -} - -func (c *cluster) waitOnStreamLeader(account, stream string) { - c.t.Helper() - expires := time.Now().Add(30 * time.Second) - for time.Now().Before(expires) { - if leader := c.streamLeader(account, stream); leader != nil { - time.Sleep(200 * time.Millisecond) - return - } - time.Sleep(100 * time.Millisecond) - } - c.t.Fatalf("Expected a stream leader for %q %q, got none", account, stream) -} - -func (c *cluster) randomNonStreamLeader(account, stream string) *Server { - c.t.Helper() - for _, s := range c.servers { - if s.JetStreamIsStreamAssigned(account, stream) && !s.JetStreamIsStreamLeader(account, stream) { - return s - } - } - return nil -} - -func (c *cluster) randomStreamNotAssigned(account, stream string) *Server { - c.t.Helper() - for _, s := range c.servers { - if !s.JetStreamIsStreamAssigned(account, stream) { - return s - } - } - return nil -} - -func (c *cluster) streamLeader(account, stream string) *Server { - c.t.Helper() - for _, s := range c.servers { - if s.JetStreamIsStreamLeader(account, stream) { - return s - } - } - return nil -} - -func (c *cluster) waitOnStreamCurrent(s *Server, account, stream string) { - c.t.Helper() - expires := time.Now().Add(30 * time.Second) - for time.Now().Before(expires) { - if s.JetStreamIsStreamCurrent(account, stream) { - time.Sleep(100 * time.Millisecond) - return - } - time.Sleep(100 * time.Millisecond) - } - c.t.Fatalf("Expected server %q to eventually be current for stream %q", s, stream) -} - -func (c *cluster) waitOnServerHealthz(s *Server) { - c.t.Helper() - expires := time.Now().Add(30 * time.Second) - for time.Now().Before(expires) { - hs := s.healthz() - if hs.Status == "ok" && hs.Error == _EMPTY_ { - return - } - time.Sleep(100 * time.Millisecond) - } - c.t.Fatalf("Expected server %q to eventually return healthz 'ok', but got %q", s, s.healthz().Error) -} - -func (c *cluster) waitOnServerCurrent(s *Server) { - c.t.Helper() - expires := time.Now().Add(20 * time.Second) - for time.Now().Before(expires) { - time.Sleep(100 * time.Millisecond) - if !s.JetStreamEnabled() || s.JetStreamIsCurrent() { - return - } - } - c.t.Fatalf("Expected server %q to eventually be current", s) -} - -func (c *cluster) waitOnAllCurrent() { - for _, cs := range c.servers { - c.waitOnServerCurrent(cs) - } -} - -func (c *cluster) serverByName(sname string) *Server { - for _, s := range c.servers { - if s.Name() == sname { - return s - } - } - return nil -} - -func (c *cluster) randomNonLeader() *Server { - // range should randomize.. but.. - for _, s := range c.servers { - if s.Running() && !s.JetStreamIsLeader() { - return s - } - } - return nil -} - -func (c *cluster) leader() *Server { - for _, s := range c.servers { - if s.JetStreamIsLeader() { - return s - } - } - return nil -} - -func (c *cluster) expectNoLeader() { - c.t.Helper() - expires := time.Now().Add(maxElectionTimeout) - for time.Now().Before(expires) { - if c.leader() == nil { - return - } - time.Sleep(10 * time.Millisecond) - } - c.t.Fatalf("Expected no leader but have one") -} - -func (c *cluster) waitOnLeader() { - c.t.Helper() - expires := time.Now().Add(40 * time.Second) - for time.Now().Before(expires) { - if leader := c.leader(); leader != nil { - time.Sleep(100 * time.Millisecond) - return - } - time.Sleep(10 * time.Millisecond) - } - - c.t.Fatalf("Expected a cluster leader, got none") -} - -// Helper function to check that a cluster is formed -func (c *cluster) waitOnClusterReady() { - c.t.Helper() - c.waitOnClusterReadyWithNumPeers(len(c.servers)) -} - -func (c *cluster) waitOnClusterReadyWithNumPeers(numPeersExpected int) { - c.t.Helper() - var leader *Server - expires := time.Now().Add(40 * time.Second) - for time.Now().Before(expires) { - if leader = c.leader(); leader != nil { - break - } - time.Sleep(50 * time.Millisecond) - } - // Now make sure we have all peers. - for leader != nil && time.Now().Before(expires) { - if len(leader.JetStreamClusterPeers()) == numPeersExpected { - time.Sleep(100 * time.Millisecond) - return - } - time.Sleep(10 * time.Millisecond) - } - - if leader == nil { - c.t.Fatalf("Failed to elect a meta-leader") - } - - peersSeen := len(leader.JetStreamClusterPeers()) - c.shutdown() - if leader == nil { - c.t.Fatalf("Expected a cluster leader and fully formed cluster, no leader") - } else { - c.t.Fatalf("Expected a fully formed cluster, only %d of %d peers seen", peersSeen, numPeersExpected) - } -} - -// Helper function to remove JetStream from a server. -func (c *cluster) removeJetStream(s *Server) { - c.t.Helper() - index := -1 - for i, cs := range c.servers { - if cs == s { - index = i - break - } - } - cf := c.opts[index].ConfigFile - cb, _ := ioutil.ReadFile(cf) - var sb strings.Builder - for _, l := range strings.Split(string(cb), "\n") { - if !strings.HasPrefix(strings.TrimSpace(l), "jetstream") { - sb.WriteString(l + "\n") - } - } - if err := ioutil.WriteFile(cf, []byte(sb.String()), 0644); err != nil { - c.t.Fatalf("Error writing updated config file: %v", err) - } - if err := s.Reload(); err != nil { - c.t.Fatalf("Error on server reload: %v", err) - } - time.Sleep(100 * time.Millisecond) -} - -func (c *cluster) stopAll() { - c.t.Helper() - for _, s := range c.servers { - s.Shutdown() - } -} - -func (c *cluster) restartAll() { - c.t.Helper() - for i, s := range c.servers { - if !s.Running() { - opts := c.opts[i] - s, o := RunServerWithConfig(opts.ConfigFile) - c.servers[i] = s - c.opts[i] = o - } - } - c.waitOnClusterReady() -} - -func (c *cluster) restartAllSamePorts() { - c.t.Helper() - for i, s := range c.servers { - if !s.Running() { - opts := c.opts[i] - s := RunServer(opts) - c.servers[i] = s - } - } - c.waitOnClusterReady() -} - -func (c *cluster) totalSubs() (total int) { - c.t.Helper() - for _, s := range c.servers { - total += int(s.NumSubscriptions()) - } - return total -} - -func (c *cluster) stableTotalSubs() (total int) { - nsubs := -1 - checkFor(c.t, 2*time.Second, 250*time.Millisecond, func() error { - subs := c.totalSubs() - if subs == nsubs { - return nil - } - nsubs = subs - return fmt.Errorf("Still stabilizing") - }) - return nsubs - -} - -func TestJetStreamMirrorSourceLoop(t *testing.T) { +func TestJetStreamClusterMirrorSourceLoop(t *testing.T) { test := func(t *testing.T, s *Server, replicas int) { nc, js := jsClientConnect(t, s) defer nc.Close() diff --git a/server/jetstream_helpers_test.go b/server/jetstream_helpers_test.go new file mode 100644 index 00000000..21129d5e --- /dev/null +++ b/server/jetstream_helpers_test.go @@ -0,0 +1,1329 @@ +// Copyright 2020-2022 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Do not exlude this file with the !skip_js_tests since those helpers +// are also used by MQTT. + +package server + +import ( + "fmt" + "io/ioutil" + "math/rand" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +// Support functions + +// Used to setup superclusters for tests. +type supercluster struct { + t *testing.T + clusters []*cluster +} + +func (sc *supercluster) shutdown() { + if sc == nil { + return + } + for _, c := range sc.clusters { + shutdownCluster(c) + } +} + +func (sc *supercluster) randomServer() *Server { + return sc.randomCluster().randomServer() +} + +func (sc *supercluster) serverByName(sname string) *Server { + for _, c := range sc.clusters { + if s := c.serverByName(sname); s != nil { + return s + } + } + return nil +} + +func (sc *supercluster) waitOnStreamLeader(account, stream string) { + sc.t.Helper() + expires := time.Now().Add(30 * time.Second) + for time.Now().Before(expires) { + for _, c := range sc.clusters { + if leader := c.streamLeader(account, stream); leader != nil { + time.Sleep(200 * time.Millisecond) + return + } + } + time.Sleep(100 * time.Millisecond) + } + sc.t.Fatalf("Expected a stream leader for %q %q, got none", account, stream) +} + +var jsClusterAccountsTempl = ` + listen: 127.0.0.1:-1 + + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + leaf { + listen: 127.0.0.1:-1 + } + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + no_auth_user: one + + accounts { + ONE { users = [ { user: "one", pass: "p" } ]; jetstream: enabled } + TWO { users = [ { user: "two", pass: "p" } ]; jetstream: enabled } + NOJS { users = [ { user: "nojs", pass: "p" } ] } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } +` + +var jsClusterTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + leaf { + listen: 127.0.0.1:-1 + } + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } +` + +var jsClusterMaxBytesTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + leaf { + listen: 127.0.0.1:-1 + } + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + no_auth_user: u + + accounts { + $U { + users = [ { user: "u", pass: "p" } ] + jetstream: { + max_mem: 128MB + max_file: 18GB + max_bytes: true // Forces streams to indicate max_bytes. + } + } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } +` + +var jsClusterMaxBytesAccountLimitTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 4GB, store_dir: '%s'} + + leaf { + listen: 127.0.0.1:-1 + } + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + no_auth_user: u + + accounts { + $U { + users = [ { user: "u", pass: "p" } ] + jetstream: { + max_mem: 128MB + max_file: 3GB + max_bytes: true // Forces streams to indicate max_bytes. + } + } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } +` + +var jsSuperClusterTempl = ` + %s + gateway { + name: %s + listen: 127.0.0.1:%d + gateways = [%s + ] + } + + system_account: "$SYS" +` + +var jsClusterLimitsTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'} + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + no_auth_user: u + + accounts { + ONE { + users = [ { user: "u", pass: "s3cr3t!" } ] + jetstream: enabled + } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } +` + +var jsMixedModeGlobalAccountTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 2MB, max_file_store: 8MB, store_dir: '%s'} + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + accounts {$SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } +` + +var jsGWTempl = `%s{name: %s, urls: [%s]}` + +func createJetStreamTaggedSuperCluster(t *testing.T) *supercluster { + sc := createJetStreamSuperCluster(t, 3, 3) + sc.waitOnPeerCount(9) + + reset := func(s *Server) { + s.mu.Lock() + rch := s.sys.resetCh + s.mu.Unlock() + if rch != nil { + rch <- struct{}{} + } + s.sendStatszUpdate() + } + + // Make first cluster AWS, US country code. + for _, s := range sc.clusterForName("C1").servers { + s.optsMu.Lock() + s.opts.Tags.Add("cloud:aws") + s.opts.Tags.Add("country:us") + s.optsMu.Unlock() + reset(s) + } + // Make second cluster GCP, UK country code. + for _, s := range sc.clusterForName("C2").servers { + s.optsMu.Lock() + s.opts.Tags.Add("cloud:gcp") + s.opts.Tags.Add("country:uk") + s.optsMu.Unlock() + reset(s) + } + // Make third cluster AZ, JP country code. + for _, s := range sc.clusterForName("C3").servers { + s.optsMu.Lock() + s.opts.Tags.Add("cloud:az") + s.opts.Tags.Add("country:jp") + s.optsMu.Unlock() + reset(s) + } + + ml := sc.leader() + js := ml.getJetStream() + require_True(t, js != nil) + js.mu.RLock() + defer js.mu.RUnlock() + cc := js.cluster + require_True(t, cc != nil) + + // Walk and make sure all tags are registered. + expires := time.Now().Add(10 * time.Second) + for time.Now().Before(expires) { + allOK := true + for _, p := range cc.meta.Peers() { + si, ok := ml.nodeToInfo.Load(p.ID) + require_True(t, ok) + ni := si.(nodeInfo) + if len(ni.tags) == 0 { + allOK = false + reset(sc.serverByName(ni.name)) + } + } + if allOK { + break + } + } + + return sc +} + +func createJetStreamSuperCluster(t *testing.T, numServersPer, numClusters int) *supercluster { + return createJetStreamSuperClusterWithTemplate(t, jsClusterTempl, numServersPer, numClusters) +} + +func createJetStreamSuperClusterWithTemplate(t *testing.T, tmpl string, numServersPer, numClusters int) *supercluster { + return createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, numServersPer, numClusters, nil) +} + +func createJetStreamSuperClusterWithTemplateAndModHook(t *testing.T, tmpl string, numServersPer, numClusters int, modify modifyCb) *supercluster { + t.Helper() + if numServersPer < 1 { + t.Fatalf("Number of servers must be >= 1") + } + if numClusters <= 1 { + t.Fatalf("Number of clusters must be > 1") + } + + startClusterPorts := []int{20_022, 22_022, 24_022} + startGatewayPorts := []int{20_122, 22_122, 24_122} + startClusterPort := startClusterPorts[rand.Intn(len(startClusterPorts))] + startGWPort := startGatewayPorts[rand.Intn(len(startGatewayPorts))] + + // Make the GWs form faster for the tests. + SetGatewaysSolicitDelay(10 * time.Millisecond) + defer ResetGatewaysSolicitDelay() + + cp, gp := startClusterPort, startGWPort + var clusters []*cluster + + var gws []string + // Build GWs first, will be same for all servers. + for i, port := 1, gp; i <= numClusters; i++ { + cn := fmt.Sprintf("C%d", i) + var urls []string + for n := 0; n < numServersPer; n++ { + urls = append(urls, fmt.Sprintf("nats-route://127.0.0.1:%d", port)) + port++ + } + gws = append(gws, fmt.Sprintf(jsGWTempl, "\n\t\t\t", cn, strings.Join(urls, ","))) + } + gwconf := strings.Join(gws, "") + + for i := 1; i <= numClusters; i++ { + cn := fmt.Sprintf("C%d", i) + // Go ahead and build configurations. + c := &cluster{servers: make([]*Server, 0, numServersPer), opts: make([]*Options, 0, numServersPer), name: cn} + + // Build out the routes that will be shared with all configs. + var routes []string + for port := cp; port < cp+numServersPer; port++ { + routes = append(routes, fmt.Sprintf("nats-route://127.0.0.1:%d", port)) + } + routeConfig := strings.Join(routes, ",") + + for si := 0; si < numServersPer; si++ { + storeDir := createDir(t, JetStreamStoreDir) + sn := fmt.Sprintf("%s-S%d", cn, si+1) + bconf := fmt.Sprintf(tmpl, sn, storeDir, cn, cp+si, routeConfig) + conf := fmt.Sprintf(jsSuperClusterTempl, bconf, cn, gp, gwconf) + gp++ + if modify != nil { + conf = modify(sn, cn, storeDir, conf) + } + s, o := RunServerWithConfig(createConfFile(t, []byte(conf))) + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + } + checkClusterFormed(t, c.servers...) + clusters = append(clusters, c) + cp += numServersPer + c.t = t + } + + // Wait for the supercluster to be formed. + egws := numClusters - 1 + for _, c := range clusters { + for _, s := range c.servers { + waitForOutboundGateways(t, s, egws, 2*time.Second) + } + } + + sc := &supercluster{t, clusters} + sc.waitOnLeader() + sc.waitOnAllCurrent() + + // Wait for all the peer nodes to be registered. + checkFor(t, 5*time.Second, 100*time.Millisecond, func() error { + var peers []string + if ml := sc.leader(); ml != nil { + peers = ml.ActivePeers() + if len(peers) == numClusters*numServersPer { + return nil + } + } + return fmt.Errorf("Not correct number of peers, expected %d, got %d", numClusters*numServersPer, len(peers)) + }) + + if sc.leader() == nil { + sc.t.Fatalf("Expected a cluster leader, got none") + } + + return sc +} + +func (sc *supercluster) createLeafNodes(clusterName string, numServers int) *cluster { + // Create our leafnode cluster template first. + return sc.createLeafNodesWithDomain(clusterName, numServers, "") +} + +func (sc *supercluster) createLeafNodesWithDomain(clusterName string, numServers int, domain string) *cluster { + // Create our leafnode cluster template first. + return sc.randomCluster().createLeafNodes(clusterName, numServers, domain) +} + +func (sc *supercluster) createSingleLeafNode(extend bool) *Server { + return sc.randomCluster().createLeafNode(extend) +} + +func (sc *supercluster) leader() *Server { + for _, c := range sc.clusters { + if leader := c.leader(); leader != nil { + return leader + } + } + return nil +} + +func (sc *supercluster) waitOnLeader() { + sc.t.Helper() + expires := time.Now().Add(5 * time.Second) + for time.Now().Before(expires) { + for _, c := range sc.clusters { + if leader := c.leader(); leader != nil { + time.Sleep(250 * time.Millisecond) + return + } + } + time.Sleep(25 * time.Millisecond) + } + sc.t.Fatalf("Expected a cluster leader, got none") +} + +func (sc *supercluster) waitOnAllCurrent() { + for _, c := range sc.clusters { + c.waitOnAllCurrent() + } +} + +func (sc *supercluster) clusterForName(name string) *cluster { + for _, c := range sc.clusters { + if c.name == name { + return c + } + } + return nil +} + +func (sc *supercluster) randomCluster() *cluster { + clusters := append(sc.clusters[:0:0], sc.clusters...) + rand.Shuffle(len(clusters), func(i, j int) { clusters[i], clusters[j] = clusters[j], clusters[i] }) + return clusters[0] +} + +func (sc *supercluster) waitOnPeerCount(n int) { + sc.t.Helper() + sc.waitOnLeader() + leader := sc.leader() + expires := time.Now().Add(20 * time.Second) + for time.Now().Before(expires) { + peers := leader.JetStreamClusterPeers() + if len(peers) == n { + return + } + time.Sleep(100 * time.Millisecond) + } + sc.t.Fatalf("Expected a super cluster peer count of %d, got %d", n, len(leader.JetStreamClusterPeers())) +} + +var jsClusterMirrorSourceImportsTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + no_auth_user: dlc + + accounts { + JS { + jetstream: enabled + users = [ { user: "rip", pass: "pass" } ] + exports [ + { service: "$JS.API.CONSUMER.>" } # To create internal consumers to mirror/source. + { stream: "RI.DELIVER.SYNC.>" } # For the mirror/source consumers sending to IA via delivery subject. + { service: "$JS.FC.>" } + ] + } + IA { + jetstream: enabled + users = [ { user: "dlc", pass: "pass" } ] + imports [ + { service: { account: JS, subject: "$JS.API.CONSUMER.>"}, to: "RI.JS.API.CONSUMER.>" } + { stream: { account: JS, subject: "RI.DELIVER.SYNC.>"} } + { service: {account: JS, subject: "$JS.FC.>" }} + ] + } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } +` + +var jsClusterImportsTempl = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + no_auth_user: dlc + + accounts { + JS { + jetstream: enabled + users = [ { user: "rip", pass: "pass" } ] + exports [ + { service: "$JS.API.>", response: stream } + { service: "TEST" } # For publishing to the stream. + { service: "$JS.ACK.TEST.*.>" } + ] + } + IA { + users = [ { user: "dlc", pass: "pass" } ] + imports [ + { service: { subject: "$JS.API.>", account: JS }} + { service: { subject: "TEST", account: JS }} + { service: { subject: "$JS.ACK.TEST.*.>", account: JS }} + ] + } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } +` + +func createMixedModeCluster(t *testing.T, tmpl string, clusterName, snPre string, numJsServers, numNonServers int, doJSConfig bool) *cluster { + t.Helper() + + if clusterName == _EMPTY_ || numJsServers < 0 || numNonServers < 1 { + t.Fatalf("Bad params") + } + + numServers := numJsServers + numNonServers + const startClusterPort = 23232 + + // Build out the routes that will be shared with all configs. + var routes []string + for cp := startClusterPort; cp < startClusterPort+numServers; cp++ { + routes = append(routes, fmt.Sprintf("nats-route://127.0.0.1:%d", cp)) + } + routeConfig := strings.Join(routes, ",") + + // Go ahead and build configurations and start servers. + c := &cluster{servers: make([]*Server, 0, numServers), opts: make([]*Options, 0, numServers), name: clusterName} + + for cp := startClusterPort; cp < startClusterPort+numServers; cp++ { + storeDir := createDir(t, JetStreamStoreDir) + + sn := fmt.Sprintf("%sS-%d", snPre, cp-startClusterPort+1) + conf := fmt.Sprintf(tmpl, sn, storeDir, clusterName, cp, routeConfig) + + // Disable JS here. + if cp-startClusterPort >= numJsServers { + // We can disable by commmenting it out, meaning no JS config, or can set the config up and just set disabled. + // e.g. jetstream: {domain: "SPOKE", enabled: false} + if doJSConfig { + conf = strings.Replace(conf, "jetstream: {", "jetstream: { enabled: false, ", 1) + } else { + conf = strings.Replace(conf, "jetstream: ", "# jetstream: ", 1) + } + } + + s, o := RunServerWithConfig(createConfFile(t, []byte(conf))) + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + } + c.t = t + + // Wait til we are formed and have a leader. + c.checkClusterFormed() + if numJsServers > 0 { + c.waitOnPeerCount(numJsServers) + } + + return c +} + +// This will create a cluster that is explicitly configured for the routes, etc. +// and also has a defined clustername. All configs for routes and cluster name will be the same. +func createJetStreamClusterExplicit(t *testing.T, clusterName string, numServers int) *cluster { + return createJetStreamClusterWithTemplate(t, jsClusterTempl, clusterName, numServers) +} + +func createJetStreamClusterWithTemplate(t *testing.T, tmpl string, clusterName string, numServers int) *cluster { + return createJetStreamClusterWithTemplateAndModHook(t, tmpl, clusterName, numServers, nil) +} + +func createJetStreamClusterWithTemplateAndModHook(t *testing.T, tmpl string, clusterName string, numServers int, modify modifyCb) *cluster { + startPorts := []int{7_022, 9_022, 11_022, 15_022} + port := startPorts[rand.Intn(len(startPorts))] + return createJetStreamClusterAndModHook(t, tmpl, clusterName, _EMPTY_, numServers, port, true, modify) +} + +func createJetStreamCluster(t *testing.T, tmpl string, clusterName, snPre string, numServers int, portStart int, waitOnReady bool) *cluster { + return createJetStreamClusterAndModHook(t, tmpl, clusterName, snPre, numServers, portStart, waitOnReady, nil) +} + +type modifyCb func(serverName, clusterName, storeDir, conf string) string + +func createJetStreamClusterAndModHook(t *testing.T, tmpl string, clusterName, snPre string, numServers int, portStart int, waitOnReady bool, modify modifyCb) *cluster { + t.Helper() + if clusterName == _EMPTY_ || numServers < 1 { + t.Fatalf("Bad params") + } + + // Flaky test prevention: + // Binding a socket to IP stack port 0 will bind an ephemeral port from an OS-specific range. + // If someone passes in to us a port spec which would cover that range, the test would be flaky. + // Adjust these ports to be the most inclusive across the port runner OSes. + // Linux: /proc/sys/net/ipv4/ip_local_port_range : 32768:60999 + // is useful, and shows there's no safe available range without OS-specific tuning. + // Our tests are usually run on Linux. Folks who care about other OSes: if you can't tune your test-runner OS to match, please + // propose a viable alternative. + const prohibitedPortFirst = 32768 + const prohibitedPortLast = 60999 + if (portStart >= prohibitedPortFirst && portStart <= prohibitedPortLast) || + (portStart+numServers-1 >= prohibitedPortFirst && portStart+numServers-1 <= prohibitedPortLast) { + t.Fatalf("test setup failure: may not specify a cluster port range which falls within %d:%d", prohibitedPortFirst, prohibitedPortLast) + } + + // Build out the routes that will be shared with all configs. + var routes []string + for cp := portStart; cp < portStart+numServers; cp++ { + routes = append(routes, fmt.Sprintf("nats-route://127.0.0.1:%d", cp)) + } + routeConfig := strings.Join(routes, ",") + + // Go ahead and build configurations and start servers. + c := &cluster{servers: make([]*Server, 0, numServers), opts: make([]*Options, 0, numServers), name: clusterName} + + for cp := portStart; cp < portStart+numServers; cp++ { + storeDir := createDir(t, JetStreamStoreDir) + sn := fmt.Sprintf("%sS-%d", snPre, cp-portStart+1) + conf := fmt.Sprintf(tmpl, sn, storeDir, clusterName, cp, routeConfig) + if modify != nil { + conf = modify(sn, clusterName, storeDir, conf) + } + s, o := RunServerWithConfig(createConfFile(t, []byte(conf))) + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + } + c.t = t + + // Wait til we are formed and have a leader. + c.checkClusterFormed() + if waitOnReady { + c.waitOnClusterReady() + } + + return c +} + +func (c *cluster) addInNewServer() *Server { + c.t.Helper() + sn := fmt.Sprintf("S-%d", len(c.servers)+1) + storeDir, _ := ioutil.TempDir(tempRoot, JetStreamStoreDir) + seedRoute := fmt.Sprintf("nats-route://127.0.0.1:%d", c.opts[0].Cluster.Port) + conf := fmt.Sprintf(jsClusterTempl, sn, storeDir, c.name, -1, seedRoute) + s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + c.checkClusterFormed() + return s +} + +// This is tied to jsClusterAccountsTempl, so changes there to users needs to be reflected here. +func (c *cluster) createSingleLeafNodeNoSystemAccount() *Server { + as := c.randomServer() + lno := as.getOpts().LeafNode + ln1 := fmt.Sprintf("nats://one:p@%s:%d", lno.Host, lno.Port) + ln2 := fmt.Sprintf("nats://two:p@%s:%d", lno.Host, lno.Port) + conf := fmt.Sprintf(jsClusterSingleLeafNodeTempl, createDir(c.t, JetStreamStoreDir), ln1, ln2) + s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + + checkLeafNodeConnectedCount(c.t, as, 2) + + return s +} + +// This is tied to jsClusterAccountsTempl, so changes there to users needs to be reflected here. +func (c *cluster) createSingleLeafNodeNoSystemAccountAndEnablesJetStream() *Server { + return c.createSingleLeafNodeNoSystemAccountAndEnablesJetStreamWithDomain(_EMPTY_, "nojs") +} + +func (c *cluster) createSingleLeafNodeNoSystemAccountAndEnablesJetStreamWithDomain(domain, user string) *Server { + tmpl := jsClusterSingleLeafNodeLikeNGSTempl + if domain != _EMPTY_ { + nsc := fmt.Sprintf("domain: %s, store_dir:", domain) + tmpl = strings.Replace(jsClusterSingleLeafNodeLikeNGSTempl, "store_dir:", nsc, 1) + } + as := c.randomServer() + lno := as.getOpts().LeafNode + ln := fmt.Sprintf("nats://%s:p@%s:%d", user, lno.Host, lno.Port) + conf := fmt.Sprintf(tmpl, createDir(c.t, JetStreamStoreDir), ln) + s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + + checkLeafNodeConnectedCount(c.t, as, 1) + + return s +} + +var jsClusterSingleLeafNodeLikeNGSTempl = ` + listen: 127.0.0.1:-1 + server_name: LNJS + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + leaf { remotes [ { urls: [ %s ] } ] } +` + +var jsClusterSingleLeafNodeTempl = ` + listen: 127.0.0.1:-1 + server_name: LNJS + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + leaf { remotes [ + { urls: [ %s ], account: "JSY" } + { urls: [ %s ], account: "JSN" } ] + } + + accounts { + JSY { users = [ { user: "y", pass: "p" } ]; jetstream: true } + JSN { users = [ { user: "n", pass: "p" } ] } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } +` + +var jsClusterTemplWithLeafNode = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + {{leaf}} + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } +` + +var jsClusterTemplWithLeafNodeNoJS = ` + listen: 127.0.0.1:-1 + server_name: %s + + # Need to keep below since it fills in the store dir by default so just comment out. + # jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + {{leaf}} + + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } +` + +var jsClusterTemplWithSingleLeafNode = ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + + {{leaf}} + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } +` + +var jsClusterTemplWithSingleLeafNodeNoJS = ` + listen: 127.0.0.1:-1 + server_name: %s + + # jetstream: {store_dir: '%s'} + + {{leaf}} + + # For access to system account. + accounts { $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } } +` + +var jsLeafFrag = ` + leaf { + remotes [ + { urls: [ %s ] } + { urls: [ %s ], account: "$SYS" } + ] + } +` + +func (c *cluster) createLeafNodes(clusterName string, numServers int, domain string) *cluster { + return c.createLeafNodesWithStartPortAndDomain(clusterName, numServers, 22111, domain) +} + +func (c *cluster) createLeafNodesNoJS(clusterName string, numServers int) *cluster { + return c.createLeafNodesWithTemplateAndStartPort(jsClusterTemplWithLeafNodeNoJS, clusterName, numServers, 21333) +} + +func (c *cluster) createLeafNodesWithStartPortAndDomain(clusterName string, numServers int, portStart int, domain string) *cluster { + if domain == _EMPTY_ { + return c.createLeafNodesWithTemplateAndStartPort(jsClusterTemplWithLeafNode, clusterName, numServers, portStart) + } + tmpl := strings.Replace(jsClusterTemplWithLeafNode, "store_dir:", fmt.Sprintf(`domain: "%s", store_dir:`, domain), 1) + return c.createLeafNodesWithTemplateAndStartPort(tmpl, clusterName, numServers, portStart) +} + +func (c *cluster) createLeafNode(extend bool) *Server { + if extend { + return c.createLeafNodeWithTemplate("LNS", + strings.ReplaceAll(jsClusterTemplWithSingleLeafNode, "store_dir:", " extension_hint: will_extend, store_dir:")) + } else { + return c.createLeafNodeWithTemplate("LNS", jsClusterTemplWithSingleLeafNode) + } +} + +func (c *cluster) createLeafNodeWithTemplate(name, template string) *Server { + tmpl := c.createLeafSolicit(template) + conf := fmt.Sprintf(tmpl, name, createDir(c.t, JetStreamStoreDir)) + s, o := RunServerWithConfig(createConfFile(c.t, []byte(conf))) + c.servers = append(c.servers, s) + c.opts = append(c.opts, o) + return s +} + +// Helper to generate the leaf solicit configs. +func (c *cluster) createLeafSolicit(tmpl string) string { + // Create our leafnode cluster template first. + var lns, lnss []string + for _, s := range c.servers { + if s.ClusterName() != c.name { + continue + } + ln := s.getOpts().LeafNode + lns = append(lns, fmt.Sprintf("nats://%s:%d", ln.Host, ln.Port)) + lnss = append(lnss, fmt.Sprintf("nats://admin:s3cr3t!@%s:%d", ln.Host, ln.Port)) + } + lnc := strings.Join(lns, ", ") + lnsc := strings.Join(lnss, ", ") + lconf := fmt.Sprintf(jsLeafFrag, lnc, lnsc) + return strings.Replace(tmpl, "{{leaf}}", lconf, 1) +} + +func (c *cluster) createLeafNodesWithTemplateMixedMode(template, clusterName string, numJsServers, numNonServers int, doJSConfig bool) *cluster { + // Create our leafnode cluster template first. + tmpl := c.createLeafSolicit(template) + pre := clusterName + "-" + + lc := createMixedModeCluster(c.t, tmpl, clusterName, pre, numJsServers, numNonServers, doJSConfig) + for _, s := range lc.servers { + checkLeafNodeConnectedCount(c.t, s, 2) + } + lc.waitOnClusterReadyWithNumPeers(numJsServers) + + return lc +} + +func (c *cluster) createLeafNodesWithTemplateAndStartPort(template, clusterName string, numServers int, portStart int) *cluster { + // Create our leafnode cluster template first. + tmpl := c.createLeafSolicit(template) + pre := clusterName + "-" + lc := createJetStreamCluster(c.t, tmpl, clusterName, pre, numServers, portStart, false) + for _, s := range lc.servers { + checkLeafNodeConnectedCount(c.t, s, 2) + } + return lc +} + +// Will add in the mapping for the account to each server. +func (c *cluster) addSubjectMapping(account, src, dest string) { + for _, s := range c.servers { + if s.ClusterName() != c.name { + continue + } + acc, err := s.LookupAccount(account) + if err != nil { + c.t.Fatalf("Unexpected error on %v: %v", s, err) + } + if err := acc.AddMapping(src, dest); err != nil { + c.t.Fatalf("Error adding mapping: %v", err) + } + } + // Make sure interest propagates. + time.Sleep(200 * time.Millisecond) +} + +// Adjust limits for the given account. +func (c *cluster) updateLimits(account string, newLimits map[string]JetStreamAccountLimits) { + c.t.Helper() + for _, s := range c.servers { + acc, err := s.LookupAccount(account) + if err != nil { + c.t.Fatalf("Unexpected error: %v", err) + } + if err := acc.UpdateJetStreamLimits(newLimits); err != nil { + c.t.Fatalf("Unexpected error: %v", err) + } + } +} + +// Hack for staticcheck +var skip = func(t *testing.T) { + t.SkipNow() +} + +func jsClientConnect(t *testing.T, s *Server, opts ...nats.Option) (*nats.Conn, nats.JetStreamContext) { + t.Helper() + nc, err := nats.Connect(s.ClientURL(), opts...) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + js, err := nc.JetStream(nats.MaxWait(10 * time.Second)) + if err != nil { + t.Fatalf("Unexpected error getting JetStream context: %v", err) + } + return nc, js +} + +func jsClientConnectEx(t *testing.T, s *Server, domain string, opts ...nats.Option) (*nats.Conn, nats.JetStreamContext) { + t.Helper() + nc, err := nats.Connect(s.ClientURL(), opts...) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + js, err := nc.JetStream(nats.MaxWait(10*time.Second), nats.Domain(domain)) + if err != nil { + t.Fatalf("Unexpected error getting JetStream context: %v", err) + } + return nc, js +} + +func checkSubsPending(t *testing.T, sub *nats.Subscription, numExpected int) { + t.Helper() + checkFor(t, 10*time.Second, 20*time.Millisecond, func() error { + if nmsgs, _, err := sub.Pending(); err != nil || nmsgs != numExpected { + return fmt.Errorf("Did not receive correct number of messages: %d vs %d", nmsgs, numExpected) + } + return nil + }) +} + +func fetchMsgs(t *testing.T, sub *nats.Subscription, numExpected int, totalWait time.Duration) []*nats.Msg { + t.Helper() + result := make([]*nats.Msg, 0, numExpected) + for start, count, wait := time.Now(), numExpected, totalWait; len(result) != numExpected; { + msgs, err := sub.Fetch(count, nats.MaxWait(wait)) + if err != nil { + t.Fatal(err) + } + result = append(result, msgs...) + count -= len(msgs) + if wait = totalWait - time.Since(start); wait < 0 { + break + } + } + if len(result) != numExpected { + t.Fatalf("Unexpected msg count, got %d, want %d", len(result), numExpected) + } + return result +} + +func (c *cluster) restartServer(rs *Server) *Server { + c.t.Helper() + index := -1 + var opts *Options + for i, s := range c.servers { + if s == rs { + index = i + break + } + } + if index < 0 { + c.t.Fatalf("Could not find server %v to restart", rs) + } + opts = c.opts[index] + s, o := RunServerWithConfig(opts.ConfigFile) + c.servers[index] = s + c.opts[index] = o + return s +} + +func (c *cluster) checkClusterFormed() { + c.t.Helper() + checkClusterFormed(c.t, c.servers...) +} + +func (c *cluster) waitOnPeerCount(n int) { + c.t.Helper() + c.waitOnLeader() + leader := c.leader() + for leader == nil { + c.waitOnLeader() + leader = c.leader() + } + expires := time.Now().Add(30 * time.Second) + for time.Now().Before(expires) { + if peers := leader.JetStreamClusterPeers(); len(peers) == n { + return + } + time.Sleep(100 * time.Millisecond) + leader = c.leader() + for leader == nil { + c.waitOnLeader() + leader = c.leader() + } + } + c.t.Fatalf("Expected a cluster peer count of %d, got %d", n, len(leader.JetStreamClusterPeers())) +} + +func (c *cluster) waitOnConsumerLeader(account, stream, consumer string) { + c.t.Helper() + expires := time.Now().Add(20 * time.Second) + for time.Now().Before(expires) { + if leader := c.consumerLeader(account, stream, consumer); leader != nil { + time.Sleep(200 * time.Millisecond) + return + } + time.Sleep(100 * time.Millisecond) + } + c.t.Fatalf("Expected a consumer leader for %q %q %q, got none", account, stream, consumer) +} + +func (c *cluster) consumerLeader(account, stream, consumer string) *Server { + c.t.Helper() + for _, s := range c.servers { + if s.JetStreamIsConsumerLeader(account, stream, consumer) { + return s + } + } + return nil +} + +func (c *cluster) randomNonConsumerLeader(account, stream, consumer string) *Server { + c.t.Helper() + for _, s := range c.servers { + if !s.JetStreamIsConsumerLeader(account, stream, consumer) { + return s + } + } + return nil +} + +func (c *cluster) waitOnStreamLeader(account, stream string) { + c.t.Helper() + expires := time.Now().Add(30 * time.Second) + for time.Now().Before(expires) { + if leader := c.streamLeader(account, stream); leader != nil { + time.Sleep(200 * time.Millisecond) + return + } + time.Sleep(100 * time.Millisecond) + } + c.t.Fatalf("Expected a stream leader for %q %q, got none", account, stream) +} + +func (c *cluster) randomNonStreamLeader(account, stream string) *Server { + c.t.Helper() + for _, s := range c.servers { + if s.JetStreamIsStreamAssigned(account, stream) && !s.JetStreamIsStreamLeader(account, stream) { + return s + } + } + return nil +} + +func (c *cluster) randomStreamNotAssigned(account, stream string) *Server { + c.t.Helper() + for _, s := range c.servers { + if !s.JetStreamIsStreamAssigned(account, stream) { + return s + } + } + return nil +} + +func (c *cluster) streamLeader(account, stream string) *Server { + c.t.Helper() + for _, s := range c.servers { + if s.JetStreamIsStreamLeader(account, stream) { + return s + } + } + return nil +} + +func (c *cluster) waitOnStreamCurrent(s *Server, account, stream string) { + c.t.Helper() + expires := time.Now().Add(30 * time.Second) + for time.Now().Before(expires) { + if s.JetStreamIsStreamCurrent(account, stream) { + time.Sleep(100 * time.Millisecond) + return + } + time.Sleep(100 * time.Millisecond) + } + c.t.Fatalf("Expected server %q to eventually be current for stream %q", s, stream) +} + +func (c *cluster) waitOnServerHealthz(s *Server) { + c.t.Helper() + expires := time.Now().Add(30 * time.Second) + for time.Now().Before(expires) { + hs := s.healthz() + if hs.Status == "ok" && hs.Error == _EMPTY_ { + return + } + time.Sleep(100 * time.Millisecond) + } + c.t.Fatalf("Expected server %q to eventually return healthz 'ok', but got %q", s, s.healthz().Error) +} + +func (c *cluster) waitOnServerCurrent(s *Server) { + c.t.Helper() + expires := time.Now().Add(20 * time.Second) + for time.Now().Before(expires) { + time.Sleep(100 * time.Millisecond) + if !s.JetStreamEnabled() || s.JetStreamIsCurrent() { + return + } + } + c.t.Fatalf("Expected server %q to eventually be current", s) +} + +func (c *cluster) waitOnAllCurrent() { + for _, cs := range c.servers { + c.waitOnServerCurrent(cs) + } +} + +func (c *cluster) serverByName(sname string) *Server { + for _, s := range c.servers { + if s.Name() == sname { + return s + } + } + return nil +} + +func (c *cluster) randomNonLeader() *Server { + // range should randomize.. but.. + for _, s := range c.servers { + if s.Running() && !s.JetStreamIsLeader() { + return s + } + } + return nil +} + +func (c *cluster) leader() *Server { + for _, s := range c.servers { + if s.JetStreamIsLeader() { + return s + } + } + return nil +} + +func (c *cluster) expectNoLeader() { + c.t.Helper() + expires := time.Now().Add(maxElectionTimeout) + for time.Now().Before(expires) { + if c.leader() == nil { + return + } + time.Sleep(10 * time.Millisecond) + } + c.t.Fatalf("Expected no leader but have one") +} + +func (c *cluster) waitOnLeader() { + c.t.Helper() + expires := time.Now().Add(40 * time.Second) + for time.Now().Before(expires) { + if leader := c.leader(); leader != nil { + time.Sleep(100 * time.Millisecond) + return + } + time.Sleep(10 * time.Millisecond) + } + + c.t.Fatalf("Expected a cluster leader, got none") +} + +// Helper function to check that a cluster is formed +func (c *cluster) waitOnClusterReady() { + c.t.Helper() + c.waitOnClusterReadyWithNumPeers(len(c.servers)) +} + +func (c *cluster) waitOnClusterReadyWithNumPeers(numPeersExpected int) { + c.t.Helper() + var leader *Server + expires := time.Now().Add(40 * time.Second) + for time.Now().Before(expires) { + if leader = c.leader(); leader != nil { + break + } + time.Sleep(50 * time.Millisecond) + } + // Now make sure we have all peers. + for leader != nil && time.Now().Before(expires) { + if len(leader.JetStreamClusterPeers()) == numPeersExpected { + time.Sleep(100 * time.Millisecond) + return + } + time.Sleep(10 * time.Millisecond) + } + + if leader == nil { + c.t.Fatalf("Failed to elect a meta-leader") + } + + peersSeen := len(leader.JetStreamClusterPeers()) + c.shutdown() + if leader == nil { + c.t.Fatalf("Expected a cluster leader and fully formed cluster, no leader") + } else { + c.t.Fatalf("Expected a fully formed cluster, only %d of %d peers seen", peersSeen, numPeersExpected) + } +} + +// Helper function to remove JetStream from a server. +func (c *cluster) removeJetStream(s *Server) { + c.t.Helper() + index := -1 + for i, cs := range c.servers { + if cs == s { + index = i + break + } + } + cf := c.opts[index].ConfigFile + cb, _ := ioutil.ReadFile(cf) + var sb strings.Builder + for _, l := range strings.Split(string(cb), "\n") { + if !strings.HasPrefix(strings.TrimSpace(l), "jetstream") { + sb.WriteString(l + "\n") + } + } + if err := ioutil.WriteFile(cf, []byte(sb.String()), 0644); err != nil { + c.t.Fatalf("Error writing updated config file: %v", err) + } + if err := s.Reload(); err != nil { + c.t.Fatalf("Error on server reload: %v", err) + } + time.Sleep(100 * time.Millisecond) +} + +func (c *cluster) stopAll() { + c.t.Helper() + for _, s := range c.servers { + s.Shutdown() + } +} + +func (c *cluster) restartAll() { + c.t.Helper() + for i, s := range c.servers { + if !s.Running() { + opts := c.opts[i] + s, o := RunServerWithConfig(opts.ConfigFile) + c.servers[i] = s + c.opts[i] = o + } + } + c.waitOnClusterReady() +} + +func (c *cluster) restartAllSamePorts() { + c.t.Helper() + for i, s := range c.servers { + if !s.Running() { + opts := c.opts[i] + s := RunServer(opts) + c.servers[i] = s + } + } + c.waitOnClusterReady() +} + +func (c *cluster) totalSubs() (total int) { + c.t.Helper() + for _, s := range c.servers { + total += int(s.NumSubscriptions()) + } + return total +} + +func (c *cluster) stableTotalSubs() (total int) { + nsubs := -1 + checkFor(c.t, 2*time.Second, 250*time.Millisecond, func() error { + subs := c.totalSubs() + if subs == nsubs { + return nil + } + nsubs = subs + return fmt.Errorf("Still stabilizing") + }) + return nsubs + +} diff --git a/server/jetstream_jwt_test.go b/server/jetstream_jwt_test.go new file mode 100644 index 00000000..c8849be2 --- /dev/null +++ b/server/jetstream_jwt_test.go @@ -0,0 +1,763 @@ +// Copyright 2020-2022 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !skip_js_tests +// +build !skip_js_tests + +package server + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + jwt "github.com/nats-io/jwt/v2" + "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" +) + +func TestJetStreamJWTLimits(t *testing.T) { + updateJwt := func(url string, creds string, pubKey string, jwt string) { + t.Helper() + c := natsConnect(t, url, nats.UserCredentials(creds)) + defer c.Close() + if msg, err := c.Request(fmt.Sprintf(accUpdateEventSubjNew, pubKey), []byte(jwt), time.Second); err != nil { + t.Fatal("error not expected in this test", err) + } else { + content := make(map[string]interface{}) + if err := json.Unmarshal(msg.Data, &content); err != nil { + t.Fatalf("%v", err) + } else if _, ok := content["data"]; !ok { + t.Fatalf("did not get an ok response got: %v", content) + } + } + } + require_IdenticalLimits := func(infoLim JetStreamAccountLimits, lim jwt.JetStreamLimits) { + t.Helper() + if int64(infoLim.MaxConsumers) != lim.Consumer || int64(infoLim.MaxStreams) != lim.Streams || + infoLim.MaxMemory != lim.MemoryStorage || infoLim.MaxStore != lim.DiskStorage { + t.Fatalf("limits do not match %v != %v", infoLim, lim) + } + } + expect_JSDisabledForAccount := func(c *nats.Conn) { + t.Helper() + if _, err := c.Request("$JS.API.INFO", nil, time.Second); err != nats.ErrTimeout && err != nats.ErrNoResponders { + t.Fatalf("Unexpected error: %v", err) + } + } + expect_InfoError := func(c *nats.Conn) { + t.Helper() + var info JSApiAccountInfoResponse + if resp, err := c.Request("$JS.API.INFO", nil, time.Second); err != nil { + t.Fatalf("Unexpected error: %v", err) + } else if err = json.Unmarshal(resp.Data, &info); err != nil { + t.Fatalf("response1 %v got error %v", string(resp.Data), err) + } else if info.Error == nil { + t.Fatalf("expected error") + } + } + validate_limits := func(c *nats.Conn, expectedLimits jwt.JetStreamLimits) { + t.Helper() + var info JSApiAccountInfoResponse + if resp, err := c.Request("$JS.API.INFO", nil, time.Second); err != nil { + t.Fatalf("Unexpected error: %v", err) + } else if err = json.Unmarshal(resp.Data, &info); err != nil { + t.Fatalf("response1 %v got error %v", string(resp.Data), err) + } else { + require_IdenticalLimits(info.Limits, expectedLimits) + } + } + // create system account + sysKp, _ := nkeys.CreateAccount() + sysPub, _ := sysKp.PublicKey() + sysUKp, _ := nkeys.CreateUser() + sysUSeed, _ := sysUKp.Seed() + uclaim := newJWTTestUserClaims() + uclaim.Subject, _ = sysUKp.PublicKey() + sysUserJwt, err := uclaim.Encode(sysKp) + require_NoError(t, err) + sysKp.Seed() + sysCreds := genCredsFile(t, sysUserJwt, sysUSeed) + // limits to apply and check + limits1 := jwt.JetStreamLimits{MemoryStorage: 1024 * 1024, DiskStorage: 2048 * 1024, Streams: 1, Consumer: 2, MaxBytesRequired: true} + // has valid limits that would fail when incorrectly applied twice + limits2 := jwt.JetStreamLimits{MemoryStorage: 4096 * 1024, DiskStorage: 8192 * 1024, Streams: 3, Consumer: 4} + // limits exceeding actual configured value of DiskStorage + limitsExceeded := jwt.JetStreamLimits{MemoryStorage: 8192 * 1024, DiskStorage: 16384 * 1024, Streams: 5, Consumer: 6} + // create account using jetstream with both limits + akp, _ := nkeys.CreateAccount() + aPub, _ := akp.PublicKey() + claim := jwt.NewAccountClaims(aPub) + claim.Limits.JetStreamLimits = limits1 + aJwt1, err := claim.Encode(oKp) + require_NoError(t, err) + claim.Limits.JetStreamLimits = limits2 + aJwt2, err := claim.Encode(oKp) + require_NoError(t, err) + claim.Limits.JetStreamLimits = limitsExceeded + aJwtLimitsExceeded, err := claim.Encode(oKp) + require_NoError(t, err) + claim.Limits.JetStreamLimits = jwt.JetStreamLimits{} // disabled + aJwt4, err := claim.Encode(oKp) + require_NoError(t, err) + // account user + uKp, _ := nkeys.CreateUser() + uSeed, _ := uKp.Seed() + uclaim = newJWTTestUserClaims() + uclaim.Subject, _ = uKp.PublicKey() + userJwt, err := uclaim.Encode(akp) + require_NoError(t, err) + userCreds := genCredsFile(t, userJwt, uSeed) + dir := createDir(t, "srv") + defer removeDir(t, dir) + conf := createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:-1 + jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb} + operator: %s + resolver: { + type: full + dir: '%s' + } + system_account: %s + `, ojwt, dir, sysPub))) + defer removeFile(t, conf) + s, opts := RunServerWithConfig(conf) + defer s.Shutdown() + port := opts.Port + updateJwt(s.ClientURL(), sysCreds, aPub, aJwt1) + c := natsConnect(t, s.ClientURL(), nats.UserCredentials(userCreds), nats.ReconnectWait(200*time.Millisecond)) + defer c.Close() + validate_limits(c, limits1) + // keep using the same connection + updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) + validate_limits(c, limits2) + // keep using the same connection but do NOT CHANGE anything. + // This tests if the jwt is applied a second time (would fail) + updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) + validate_limits(c, limits2) + // keep using the same connection. This update EXCEEDS LIMITS + updateJwt(s.ClientURL(), sysCreds, aPub, aJwtLimitsExceeded) + validate_limits(c, limits2) + // disable test after failure + updateJwt(s.ClientURL(), sysCreds, aPub, aJwt4) + expect_InfoError(c) + // re enable, again testing with a value that can't be applied twice + updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) + validate_limits(c, limits2) + // disable test no prior failure + updateJwt(s.ClientURL(), sysCreds, aPub, aJwt4) + expect_InfoError(c) + // Wrong limits form start + updateJwt(s.ClientURL(), sysCreds, aPub, aJwtLimitsExceeded) + expect_JSDisabledForAccount(c) + // enable js but exceed limits. Followed by fix via restart + updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) + validate_limits(c, limits2) + updateJwt(s.ClientURL(), sysCreds, aPub, aJwtLimitsExceeded) + validate_limits(c, limits2) + s.Shutdown() + conf = createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:%d + jetstream: {max_mem_store: 20Mb, max_file_store: 20Mb} + operator: %s + resolver: { + type: full + dir: '%s' + } + system_account: %s + `, port, ojwt, dir, sysPub))) + defer removeFile(t, conf) + s, _ = RunServerWithConfig(conf) + defer s.Shutdown() + c.Flush() // force client to discover the disconnect + checkClientsCount(t, s, 1) + validate_limits(c, limitsExceeded) + s.Shutdown() + // disable jetstream test + conf = createConfFile(t, []byte(fmt.Sprintf(` + listen: 127.0.0.1:%d + operator: %s + resolver: { + type: full + dir: '%s' + } + system_account: %s + `, port, ojwt, dir, sysPub))) + defer removeFile(t, conf) + s, _ = RunServerWithConfig(conf) + defer s.Shutdown() + c.Flush() // force client to discover the disconnect + checkClientsCount(t, s, 1) + expect_JSDisabledForAccount(c) + // test that it stays disabled + updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) + expect_JSDisabledForAccount(c) + c.Close() +} + +func TestJetStreamJWTMoveWithTiers(t *testing.T) { + _, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + + accKp, aExpPub := createKey(t) + accClaim := jwt.NewAccountClaims(aExpPub) + accClaim.Name = "acc" + accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ + DiskStorage: 1100, Consumer: 1, Streams: 1} + accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ + DiskStorage: 3300, Consumer: 1, Streams: 1} + accJwt := encodeClaim(t, accClaim, aExpPub) + accCreds := newUser(t, accKp) + + test := func(t *testing.T, replicas int) { + tmlp := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + leaf { + listen: 127.0.0.1:-1 + } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + ` + s := createJetStreamSuperClusterWithTemplateAndModHook(t, tmlp, 3, 3, + func(serverName, clustername, storeDir, conf string) string { + return conf + fmt.Sprintf(` + server_tags: [cloud:%s-tag] + operator: %s + system_account: %s + resolver = MEMORY + resolver_preload = { + %s : %s + %s : %s + } + `, clustername, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt) + }) + defer s.shutdown() + + nc := natsConnect(t, s.randomServer().ClientURL(), nats.UserCredentials(accCreds)) + defer nc.Close() + + js, err := nc.JetStream() + require_NoError(t, err) + + ci, err := js.AddStream(&nats.StreamConfig{Name: "MOVE-ME", Replicas: replicas, + Placement: &nats.Placement{Tags: []string{"cloud:C1-tag"}}}) + require_NoError(t, err) + require_Equal(t, ci.Cluster.Name, "C1") + ci, err = js.UpdateStream(&nats.StreamConfig{Name: "MOVE-ME", Replicas: replicas, + Placement: &nats.Placement{Tags: []string{"cloud:C2-tag"}}}) + require_NoError(t, err) + require_Equal(t, ci.Cluster.Name, "C1") + + checkFor(t, 10*time.Second, 10*time.Millisecond, func() error { + if si, err := js.StreamInfo("MOVE-ME"); err != nil { + return err + } else if si.Cluster.Name != "C2" { + return fmt.Errorf("Wrong cluster: %q", si.Cluster.Name) + } else if si.Cluster.Leader == _EMPTY_ { + return fmt.Errorf("No leader yet") + } else if !strings.HasPrefix(si.Cluster.Leader, "C2-") { + return fmt.Errorf("Wrong leader: %q", si.Cluster.Leader) + } else if len(si.Cluster.Replicas) != replicas-1 { + return fmt.Errorf("Expected %d replicas, got %d", replicas-1, len(si.Cluster.Replicas)) + } + return nil + }) + } + + t.Run("R1", func(t *testing.T) { + test(t, 1) + }) + t.Run("R3", func(t *testing.T) { + test(t, 3) + }) +} + +func TestJetStreamJWTClusteredTiers(t *testing.T) { + sysKp, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + sysCreds := newUser(t, sysKp) + defer removeFile(t, sysCreds) + + accKp, aExpPub := createKey(t) + accClaim := jwt.NewAccountClaims(aExpPub) + accClaim.Name = "acc" + accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ + DiskStorage: 1100, Consumer: 2, Streams: 2} + accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ + DiskStorage: 3300, Consumer: 1, Streams: 1} + accJwt := encodeClaim(t, accClaim, aExpPub) + accCreds := newUser(t, accKp) + tmlp := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + leaf { + listen: 127.0.0.1:-1 + } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + ` + fmt.Sprintf(` + operator: %s + system_account: %s + resolver = MEMORY + resolver_preload = { + %s : %s + %s : %s + } + `, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt) + + c := createJetStreamClusterWithTemplate(t, tmlp, "cluster", 3) + defer c.shutdown() + + nc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds)) + defer nc.Close() + + js, err := nc.JetStream() + require_NoError(t, err) + + // Test absent tiers + _, err = js.AddStream(&nats.StreamConfig{Name: "testR2", Replicas: 2, Subjects: []string{"testR2"}}) + require_Error(t, err) + require_Equal(t, err.Error(), "no JetStream default or applicable tiered limit present") + _, err = js.AddStream(&nats.StreamConfig{Name: "testR5", Replicas: 5, Subjects: []string{"testR5"}}) + require_Error(t, err) + require_Equal(t, err.Error(), "no JetStream default or applicable tiered limit present") + + // Test tiers up to stream limits + _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-1", Replicas: 1, Subjects: []string{"testR1-1"}}) + require_NoError(t, err) + _, err = js.AddStream(&nats.StreamConfig{Name: "testR3-1", Replicas: 3, Subjects: []string{"testR3-1"}}) + require_NoError(t, err) + _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-2", Replicas: 1, Subjects: []string{"testR1-2"}}) + require_NoError(t, err) + + // Test exceeding tiered stream limit + _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-3", Replicas: 1, Subjects: []string{"testR1-3"}}) + require_Error(t, err) + require_Equal(t, err.Error(), "maximum number of streams reached") + _, err = js.AddStream(&nats.StreamConfig{Name: "testR3-3", Replicas: 3, Subjects: []string{"testR3-3"}}) + require_Error(t, err) + require_Equal(t, err.Error(), "maximum number of streams reached") + + // Test tiers up to consumer limits + _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur1", AckPolicy: nats.AckExplicitPolicy}) + require_NoError(t, err) + _, err = js.AddConsumer("testR3-1", &nats.ConsumerConfig{Durable: "dur2", AckPolicy: nats.AckExplicitPolicy}) + require_NoError(t, err) + _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur3", AckPolicy: nats.AckExplicitPolicy}) + require_NoError(t, err) + + // test exceeding tiered consumer limits + _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur4", AckPolicy: nats.AckExplicitPolicy}) + require_Error(t, err) + require_Equal(t, err.Error(), "maximum consumers limit reached") + _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur5", AckPolicy: nats.AckExplicitPolicy}) + require_Error(t, err) + require_Equal(t, err.Error(), "maximum consumers limit reached") + + // test tiered storage limit + msg := [512]byte{} + _, err = js.Publish("testR1-1", msg[:]) + require_NoError(t, err) + _, err = js.Publish("testR3-1", msg[:]) + require_NoError(t, err) + _, err = js.Publish("testR3-1", msg[:]) + require_NoError(t, err) + _, err = js.Publish("testR1-2", msg[:]) + require_NoError(t, err) + + time.Sleep(2000 * time.Millisecond) // wait for update timer to synchronize totals + + // test exceeding tiered storage limit + _, err = js.Publish("testR1-1", []byte("1")) + require_Error(t, err) + require_Equal(t, err.Error(), "nats: resource limits exceeded for account") + _, err = js.Publish("testR3-1", []byte("fail this message!")) + require_Error(t, err) + require_Equal(t, err.Error(), "nats: resource limits exceeded for account") + + // retrieve limits + var info JSApiAccountInfoResponse + m, err := nc.Request("$JS.API.INFO", nil, time.Second) + require_NoError(t, err) + err = json.Unmarshal(m.Data, &info) + require_NoError(t, err) + + require_True(t, info.Memory == 0) + // R1 streams fail message with an add followed by remove, if the update was sent in between, the count is > limit + // Alternative to checking both values is, prior to the info request, wait for another update + require_True(t, info.Store == 4400 || info.Store == 4439) + require_True(t, info.Streams == 3) + require_True(t, info.Consumers == 3) + require_True(t, info.Limits == JetStreamAccountLimits{}) + r1 := info.Tiers["R1"] + require_True(t, r1.Streams == 2) + require_True(t, r1.Consumers == 2) + // R1 streams fail message with an add followed by remove, if the update was sent in between, the count is > limit + // Alternative to checking both values is, prior to the info request, wait for another update + require_True(t, r1.Store == 1100 || r1.Store == 1139) + require_True(t, r1.Memory == 0) + require_True(t, r1.Limits == JetStreamAccountLimits{ + MaxMemory: 0, + MaxStore: 1100, + MaxStreams: 2, + MaxConsumers: 2, + MaxAckPending: -1, + MemoryMaxStreamBytes: -1, + StoreMaxStreamBytes: -1, + MaxBytesRequired: false, + }) + r3 := info.Tiers["R3"] + require_True(t, r3.Streams == 1) + require_True(t, r3.Consumers == 1) + require_True(t, r3.Store == 3300) + require_True(t, r3.Memory == 0) + require_True(t, r3.Limits == JetStreamAccountLimits{ + MaxMemory: 0, + MaxStore: 3300, + MaxStreams: 1, + MaxConsumers: 1, + MaxAckPending: -1, + MemoryMaxStreamBytes: -1, + StoreMaxStreamBytes: -1, + MaxBytesRequired: false, + }) +} + +func TestJetStreamJWTClusteredTiersChange(t *testing.T) { + sysKp, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + sysCreds := newUser(t, sysKp) + defer removeFile(t, sysCreds) + + accKp, aExpPub := createKey(t) + accClaim := jwt.NewAccountClaims(aExpPub) + accClaim.Name = "acc" + accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ + DiskStorage: 1000, MemoryStorage: 0, Consumer: 1, Streams: 1} + accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ + DiskStorage: 1500, MemoryStorage: 0, Consumer: 1, Streams: 1} + accJwt1 := encodeClaim(t, accClaim, aExpPub) + accCreds := newUser(t, accKp) + start := time.Now() + + tmlp := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + leaf { + listen: 127.0.0.1:-1 + } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + ` + c := createJetStreamClusterWithTemplateAndModHook(t, tmlp, "cluster", 3, + func(serverName, clustername, storeDir, conf string) string { + return conf + fmt.Sprintf(` + operator: %s + system_account: %s + resolver: { + type: full + dir: '%s/jwt' + }`, ojwt, syspub, storeDir) + }) + defer c.shutdown() + + updateJwt(t, c.randomServer().ClientURL(), sysCreds, sysJwt, 3) + updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt1, 3) + + nc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds)) + defer nc.Close() + + js, err := nc.JetStream() + require_NoError(t, err) + + // Test tiers up to stream limits + cfg := &nats.StreamConfig{Name: "testR1-1", Replicas: 1, Subjects: []string{"testR1-1"}, MaxBytes: 1000} + _, err = js.AddStream(cfg) + require_NoError(t, err) + + cfg.Replicas = 3 + _, err = js.UpdateStream(cfg) + require_Error(t, err) + require_Equal(t, err.Error(), "insufficient storage resources available") + + time.Sleep(time.Second - time.Since(start)) // make sure the time stamp changes + accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ + DiskStorage: 3000, MemoryStorage: 0, Consumer: 1, Streams: 1} + accJwt2 := encodeClaim(t, accClaim, aExpPub) + + updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt2, 3) + + var rBefore, rAfter JSApiAccountInfoResponse + m, err := nc.Request("$JS.API.INFO", nil, time.Second) + require_NoError(t, err) + err = json.Unmarshal(m.Data, &rBefore) + require_NoError(t, err) + _, err = js.UpdateStream(cfg) + require_NoError(t, err) + + m, err = nc.Request("$JS.API.INFO", nil, time.Second) + require_NoError(t, err) + err = json.Unmarshal(m.Data, &rAfter) + require_NoError(t, err) + require_True(t, rBefore.Tiers["R1"].Streams == 1) + require_True(t, rBefore.Tiers["R1"].Streams == rAfter.Tiers["R3"].Streams) + require_True(t, rBefore.Tiers["R3"].Streams == 0) + require_True(t, rAfter.Tiers["R1"].Streams == 0) +} + +func TestJetStreamJWTClusteredDeleteTierWithStreamAndMove(t *testing.T) { + sysKp, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + sysCreds := newUser(t, sysKp) + defer removeFile(t, sysCreds) + + accKp, aExpPub := createKey(t) + accClaim := jwt.NewAccountClaims(aExpPub) + accClaim.Name = "acc" + accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ + DiskStorage: 1000, MemoryStorage: 0, Consumer: 1, Streams: 1} + accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ + DiskStorage: 3000, MemoryStorage: 0, Consumer: 1, Streams: 1} + accJwt1 := encodeClaim(t, accClaim, aExpPub) + accCreds := newUser(t, accKp) + start := time.Now() + + tmlp := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + leaf { + listen: 127.0.0.1:-1 + } + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + ` + c := createJetStreamClusterWithTemplateAndModHook(t, tmlp, "cluster", 3, + func(serverName, clustername, storeDir, conf string) string { + return conf + fmt.Sprintf(` + operator: %s + system_account: %s + resolver: { + type: full + dir: '%s/jwt' + }`, ojwt, syspub, storeDir) + }) + defer c.shutdown() + + updateJwt(t, c.randomServer().ClientURL(), sysCreds, sysJwt, 3) + updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt1, 3) + + nc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds)) + defer nc.Close() + + js, err := nc.JetStream() + require_NoError(t, err) + + // Test tiers up to stream limits + cfg := &nats.StreamConfig{Name: "testR1-1", Replicas: 1, Subjects: []string{"testR1-1"}, MaxBytes: 1000} + _, err = js.AddStream(cfg) + require_NoError(t, err) + + _, err = js.Publish("testR1-1", nil) + require_NoError(t, err) + + time.Sleep(time.Second - time.Since(start)) // make sure the time stamp changes + delete(accClaim.Limits.JetStreamTieredLimits, "R1") + accJwt2 := encodeClaim(t, accClaim, aExpPub) + updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt2, 3) + + var respBefore JSApiAccountInfoResponse + m, err := nc.Request("$JS.API.INFO", nil, time.Second) + require_NoError(t, err) + err = json.Unmarshal(m.Data, &respBefore) + require_NoError(t, err) + + require_True(t, respBefore.JetStreamAccountStats.Tiers["R3"].Streams == 0) + require_True(t, respBefore.JetStreamAccountStats.Tiers["R1"].Streams == 1) + + _, err = js.Publish("testR1-1", nil) + require_Error(t, err) + require_Equal(t, err.Error(), "nats: no JetStream default or applicable tiered limit present") + + cfg.Replicas = 3 + _, err = js.UpdateStream(cfg) + require_NoError(t, err) + + // I noticed this taking > 5 seconds + checkFor(t, 10*time.Second, 250*time.Millisecond, func() error { + _, err = js.Publish("testR1-1", nil) + return err + }) + + var respAfter JSApiAccountInfoResponse + m, err = nc.Request("$JS.API.INFO", nil, time.Second) + require_NoError(t, err) + err = json.Unmarshal(m.Data, &respAfter) + require_NoError(t, err) + + require_True(t, respAfter.JetStreamAccountStats.Tiers["R3"].Streams == 1) + require_True(t, respAfter.JetStreamAccountStats.Tiers["R3"].Store > 0) + + _, ok := respAfter.JetStreamAccountStats.Tiers["R1"] + require_True(t, !ok) +} + +func TestJetStreamJWTSysAccUpdateMixedMode(t *testing.T) { + skp, spub := createKey(t) + sUsr := createUserCreds(t, nil, skp) + sysClaim := jwt.NewAccountClaims(spub) + sysClaim.Name = "SYS" + sysJwt := encodeClaim(t, sysClaim, spub) + encodeJwt1Time := time.Now() + + akp, apub := createKey(t) + aUsr := createUserCreds(t, nil, akp) + claim := jwt.NewAccountClaims(apub) + claim.Limits.JetStreamLimits.DiskStorage = 1024 * 1024 + claim.Limits.JetStreamLimits.Streams = 1 + jwt1 := encodeClaim(t, claim, apub) + + basePath := "/ngs/v1/accounts/jwt/" + reqCount := int32(0) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == basePath { + w.Write([]byte("ok")) + } else if strings.HasSuffix(r.URL.Path, spub) { + w.Write([]byte(sysJwt)) + } else if strings.HasSuffix(r.URL.Path, apub) { + w.Write([]byte(jwt1)) + } else { + // only count requests that could be filled + return + } + atomic.AddInt32(&reqCount, 1) + })) + defer ts.Close() + + tmpl := ` + listen: 127.0.0.1:-1 + server_name: %s + jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} + cluster { + name: %s + listen: 127.0.0.1:%d + routes = [%s] + } + ` + + sc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 3, 2, + func(serverName, clusterName, storeDir, conf string) string { + // create an ngs like setup, with connection and non connection server + if clusterName == "C1" { + conf = strings.ReplaceAll(conf, "jetstream", "#jetstream") + } + return fmt.Sprintf(`%s + operator: %s + system_account: %s + resolver: URL("%s%s")`, conf, ojwt, spub, ts.URL, basePath) + }) + defer sc.shutdown() + disconnectChan := make(chan struct{}, 100) + defer close(disconnectChan) + disconnectCb := nats.DisconnectErrHandler(func(conn *nats.Conn, err error) { + disconnectChan <- struct{}{} + }) + + s := sc.clusterForName("C1").randomServer() + + sysNc := natsConnect(t, s.ClientURL(), sUsr, disconnectCb, nats.NoCallbacksAfterClientClose()) + defer sysNc.Close() + aNc := natsConnect(t, s.ClientURL(), aUsr, disconnectCb, nats.NoCallbacksAfterClientClose()) + defer aNc.Close() + + js, err := aNc.JetStream() + require_NoError(t, err) + + si, err := js.AddStream(&nats.StreamConfig{Name: "bar", Subjects: []string{"bar"}, Replicas: 3}) + require_NoError(t, err) + require_Equal(t, si.Cluster.Name, "C2") + _, err = js.AccountInfo() + require_NoError(t, err) + + r, err := sysNc.Request(fmt.Sprintf(serverPingReqSubj, "ACCOUNTZ"), + []byte(fmt.Sprintf(`{"account":"%s"}`, spub)), time.Second) + require_NoError(t, err) + respb := ServerAPIResponse{Data: &Accountz{}} + require_NoError(t, json.Unmarshal(r.Data, &respb)) + + hasJSExp := func(resp *ServerAPIResponse) bool { + found := false + for _, e := range resp.Data.(*Accountz).Account.Exports { + if e.Subject == jsAllAPI { + found = true + break + } + } + return found + } + require_True(t, hasJSExp(&respb)) + + // make sure jti increased + time.Sleep(time.Second - time.Since(encodeJwt1Time)) + sysJwt2 := encodeClaim(t, sysClaim, spub) + + oldRcount := atomic.LoadInt32(&reqCount) + _, err = sysNc.Request(fmt.Sprintf(accUpdateEventSubjNew, spub), []byte(sysJwt2), time.Second) + require_NoError(t, err) + // test to make sure connected client (aNc) was not kicked + time.Sleep(200 * time.Millisecond) + require_True(t, len(disconnectChan) == 0) + + // ensure nothing new has happened, lookup for account not found is skipped during inc + require_True(t, atomic.LoadInt32(&reqCount) == oldRcount) + // no responders + _, err = aNc.Request("foo", nil, time.Second) + require_Error(t, err) + require_Equal(t, err.Error(), "nats: no responders available for request") + + nc2, js2 := jsClientConnect(t, sc.clusterForName("C2").randomServer(), aUsr) + defer nc2.Close() + _, err = js2.AccountInfo() + require_NoError(t, err) + + r, err = sysNc.Request(fmt.Sprintf(serverPingReqSubj, "ACCOUNTZ"), + []byte(fmt.Sprintf(`{"account":"%s"}`, spub)), time.Second) + require_NoError(t, err) + respa := ServerAPIResponse{Data: &Accountz{}} + require_NoError(t, json.Unmarshal(r.Data, &respa)) + require_True(t, hasJSExp(&respa)) + + _, err = js.AccountInfo() + require_NoError(t, err) +} diff --git a/server/jetstream_leafnode_test.go b/server/jetstream_leafnode_test.go new file mode 100644 index 00000000..64005041 --- /dev/null +++ b/server/jetstream_leafnode_test.go @@ -0,0 +1,1208 @@ +// Copyright 2020-2022 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !skip_js_tests +// +build !skip_js_tests + +package server + +import ( + "fmt" + "io/ioutil" + "os" + "strings" + "testing" + "time" + + jwt "github.com/nats-io/jwt/v2" + "github.com/nats-io/nats.go" + "github.com/nats-io/nkeys" +) + +func TestJetStreamLeafNodeUniqueServerNameCrossJSDomain(t *testing.T) { + name := "NOT-UNIQUE" + test := func(s *Server, sIdExpected string, srvs ...*Server) { + ids := map[string]string{} + for _, srv := range srvs { + checkLeafNodeConnectedCount(t, srv, 2) + ids[srv.ID()] = srv.opts.JetStreamDomain + } + // ensure that an update for every server was received + sysNc := natsConnect(t, fmt.Sprintf("nats://admin:s3cr3t!@127.0.0.1:%d", s.opts.Port)) + defer sysNc.Close() + sub, err := sysNc.SubscribeSync(fmt.Sprintf(serverStatsSubj, "*")) + require_NoError(t, err) + for { + m, err := sub.NextMsg(time.Second) + require_NoError(t, err) + tk := strings.Split(m.Subject, ".") + if domain, ok := ids[tk[2]]; ok { + delete(ids, tk[2]) + require_Contains(t, string(m.Data), fmt.Sprintf(`"domain":"%s"`, domain)) + } + if len(ids) == 0 { + break + } + } + cnt := 0 + s.nodeToInfo.Range(func(key, value interface{}) bool { + cnt++ + require_Equal(t, value.(nodeInfo).name, name) + require_Equal(t, value.(nodeInfo).id, sIdExpected) + return true + }) + require_True(t, cnt == 1) + } + tmplA := ` + listen: -1 + server_name: %s + jetstream { + max_mem_store: 256MB, + max_file_store: 2GB, + store_dir: '%s', + domain: hub + } + accounts { + JSY { users = [ { user: "y", pass: "p" } ]; jetstream: true } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } + leaf { + port: -1 + } + ` + tmplL := ` + listen: -1 + server_name: %s + jetstream { + max_mem_store: 256MB, + max_file_store: 2GB, + store_dir: '%s', + domain: %s + } + accounts { + JSY { users = [ { user: "y", pass: "p" } ]; jetstream: true } + $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } + } + leaf { + remotes [ + { urls: [ %s ], account: "JSY" } + { urls: [ %s ], account: "$SYS" } + ] + } + ` + t.Run("same-domain", func(t *testing.T) { + confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, name, createDir(t, JetStreamStoreDir)))) + defer removeFile(t, confA) + sA, oA := RunServerWithConfig(confA) + defer sA.Shutdown() + // using same domain as sA + confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, name, createDir(t, JetStreamStoreDir), "hub", + fmt.Sprintf("nats://y:p@127.0.0.1:%d", oA.LeafNode.Port), + fmt.Sprintf("nats://admin:s3cr3t!@127.0.0.1:%d", oA.LeafNode.Port)))) + defer removeFile(t, confL) + sL, _ := RunServerWithConfig(confL) + defer sL.Shutdown() + // as server name uniqueness is violates, sL.ID() is the expected value + test(sA, sL.ID(), sA, sL) + }) + t.Run("different-domain", func(t *testing.T) { + confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, name, createDir(t, JetStreamStoreDir)))) + defer removeFile(t, confA) + sA, oA := RunServerWithConfig(confA) + defer sA.Shutdown() + // using different domain as sA + confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, name, createDir(t, JetStreamStoreDir), "spoke", + fmt.Sprintf("nats://y:p@127.0.0.1:%d", oA.LeafNode.Port), + fmt.Sprintf("nats://admin:s3cr3t!@127.0.0.1:%d", oA.LeafNode.Port)))) + defer removeFile(t, confL) + sL, _ := RunServerWithConfig(confL) + defer sL.Shutdown() + checkLeafNodeConnectedCount(t, sL, 2) + checkLeafNodeConnectedCount(t, sA, 2) + // ensure sA contains only sA.ID + test(sA, sA.ID(), sA, sL) + }) +} + +func TestJetStreamLeafNodeJwtPermsAndJSDomains(t *testing.T) { + createAcc := func(js bool) (string, string, nkeys.KeyPair) { + kp, _ := nkeys.CreateAccount() + aPub, _ := kp.PublicKey() + claim := jwt.NewAccountClaims(aPub) + if js { + claim.Limits.JetStreamLimits = jwt.JetStreamLimits{ + MemoryStorage: 1024 * 1024, + DiskStorage: 1024 * 1024, + Streams: 1, Consumer: 2} + } + aJwt, err := claim.Encode(oKp) + require_NoError(t, err) + return aPub, aJwt, kp + } + sysPub, sysJwt, sysKp := createAcc(false) + accPub, accJwt, accKp := createAcc(true) + noExpiration := time.Now().Add(time.Hour) + // create user for acc to be used in leaf node. + lnCreds := createUserWithLimit(t, accKp, noExpiration, func(j *jwt.UserPermissionLimits) { + j.Sub.Deny.Add("subdeny") + j.Pub.Deny.Add("pubdeny") + }) + defer removeFile(t, lnCreds) + unlimitedCreds := createUserWithLimit(t, accKp, noExpiration, nil) + defer removeFile(t, unlimitedCreds) + + sysCreds := createUserWithLimit(t, sysKp, noExpiration, nil) + defer removeFile(t, sysCreds) + + tmplA := ` +operator: %s +system_account: %s +resolver: MEMORY +resolver_preload: { + %s: %s + %s: %s +} +listen: 127.0.0.1:-1 +leafnodes: { + listen: 127.0.0.1:-1 +} +jetstream :{ + domain: "cluster" + store_dir: '%s' + max_mem: 100Mb + max_file: 100Mb +} +` + + tmplL := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account = SYS +jetstream: { + domain: ln1 + store_dir: '%s' + max_mem: 50Mb + max_file: 50Mb +} +leafnodes:{ + remotes:[{ url:nats://127.0.0.1:%d, account: A, credentials: '%s'}, + { url:nats://127.0.0.1:%d, account: SYS, credentials: '%s'}] +} +` + + confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, ojwt, sysPub, + sysPub, sysJwt, accPub, accJwt, + createDir(t, JetStreamStoreDir)))) + defer removeFile(t, confA) + sA, _ := RunServerWithConfig(confA) + defer sA.Shutdown() + + confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, createDir(t, JetStreamStoreDir), + sA.opts.LeafNode.Port, lnCreds, sA.opts.LeafNode.Port, sysCreds))) + defer removeFile(t, confL) + sL, _ := RunServerWithConfig(confL) + defer sL.Shutdown() + + checkLeafNodeConnectedCount(t, sA, 2) + checkLeafNodeConnectedCount(t, sL, 2) + + ncA := natsConnect(t, sA.ClientURL(), nats.UserCredentials(unlimitedCreds)) + defer ncA.Close() + + ncL := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sL.opts.Port)) + defer ncL.Close() + + test := func(subject string, cSub, cPub *nats.Conn, remoteServerForSub *Server, accName string, pass bool) { + t.Helper() + sub, err := cSub.SubscribeSync(subject) + require_NoError(t, err) + require_NoError(t, cSub.Flush()) + // ensure the subscription made it across, or if not sent due to sub deny, make sure it could have made it. + if remoteServerForSub == nil { + time.Sleep(200 * time.Millisecond) + } else { + checkSubInterest(t, remoteServerForSub, accName, subject, time.Second) + } + require_NoError(t, cPub.Publish(subject, []byte("hello world"))) + require_NoError(t, cPub.Flush()) + m, err := sub.NextMsg(500 * time.Millisecond) + if pass { + require_NoError(t, err) + require_True(t, m.Subject == subject) + require_Equal(t, string(m.Data), "hello world") + } else { + require_True(t, err == nats.ErrTimeout) + } + } + + t.Run("sub-on-ln-pass", func(t *testing.T) { + test("sub", ncL, ncA, sA, accPub, true) + }) + t.Run("sub-on-ln-fail", func(t *testing.T) { + test("subdeny", ncL, ncA, nil, "", false) + }) + t.Run("pub-on-ln-pass", func(t *testing.T) { + test("pub", ncA, ncL, sL, "A", true) + }) + t.Run("pub-on-ln-fail", func(t *testing.T) { + test("pubdeny", ncA, ncL, nil, "A", false) + }) +} + +func TestJetStreamLeafNodeClusterExtensionWithSystemAccount(t *testing.T) { + /* + Topologies tested here + same == true + A <-> B + ^ |\ + | \ + | proxy + | \ + LA <-> LB + + same == false + A <-> B + ^ ^ + | | + | proxy + | | + LA <-> LB + + The proxy is turned on later, such that the system account connection can be started later, in a controlled way + This explicitly tests the system state before and after this happens. + */ + + tmplA := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account: SYS +leafnodes: { + listen: 127.0.0.1:-1 + no_advertise: true + authorization: { + timeout: 0.5 + } +} +jetstream :{ + domain: "cluster" + store_dir: '%s' + max_mem: 100Mb + max_file: 100Mb +} +server_name: A +cluster: { + name: clust1 + listen: 127.0.0.1:50554 + routes=[nats-route://127.0.0.1:50555] + no_advertise: true +} +` + + tmplB := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account: SYS +leafnodes: { + listen: 127.0.0.1:-1 + no_advertise: true + authorization: { + timeout: 0.5 + } +} +jetstream: { + domain: "cluster" + store_dir: '%s' + max_mem: 100Mb + max_file: 100Mb +} +server_name: B +cluster: { + name: clust1 + listen: 127.0.0.1:50555 + routes=[nats-route://127.0.0.1:50554] + no_advertise: true +} +` + + tmplLA := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account = SYS +jetstream: { + domain: "cluster" + store_dir: '%s' + max_mem: 50Mb + max_file: 50Mb + %s +} +server_name: LA +cluster: { + name: clustL + listen: 127.0.0.1:50556 + routes=[nats-route://127.0.0.1:50557] + no_advertise: true +} +leafnodes:{ + no_advertise: true + remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A}, + {url:nats://s1:s1@127.0.0.1:%d, account: SYS}] +} +` + + tmplLB := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account = SYS +jetstream: { + domain: "cluster" + store_dir: '%s' + max_mem: 50Mb + max_file: 50Mb + %s +} +server_name: LB +cluster: { + name: clustL + listen: 127.0.0.1:50557 + routes=[nats-route://127.0.0.1:50556] + no_advertise: true +} +leafnodes:{ + no_advertise: true + remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A}, + {url:nats://s1:s1@127.0.0.1:%d, account: SYS}] +} +` + + for _, testCase := range []struct { + // which topology to pick + same bool + // If leaf server should be operational and form a Js cluster prior to joining. + // In this setup this would be an error as you give the wrong hint. + // But this should work itself out regardless + leafFunctionPreJoin bool + }{ + {true, true}, + {true, false}, + {false, true}, + {false, false}} { + t.Run(fmt.Sprintf("%t-%t", testCase.same, testCase.leafFunctionPreJoin), func(t *testing.T) { + sd1 := createDir(t, JetStreamStoreDir) + defer os.RemoveAll(sd1) + confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, sd1))) + defer removeFile(t, confA) + sA, _ := RunServerWithConfig(confA) + defer sA.Shutdown() + + sd2 := createDir(t, JetStreamStoreDir) + defer os.RemoveAll(sd2) + confB := createConfFile(t, []byte(fmt.Sprintf(tmplB, sd2))) + defer removeFile(t, confB) + sB, _ := RunServerWithConfig(confB) + defer sB.Shutdown() + + checkClusterFormed(t, sA, sB) + + c := cluster{t: t, servers: []*Server{sA, sB}} + c.waitOnLeader() + + // starting this will allow the second remote in tmplL to successfully connect. + port := sB.opts.LeafNode.Port + if testCase.same { + port = sA.opts.LeafNode.Port + } + p := &proxyAcceptDetectFailureLate{acceptPort: port} + defer p.close() + lPort := p.runEx(t, true) + + hint := "" + if testCase.leafFunctionPreJoin { + hint = fmt.Sprintf("extension_hint: %s", strings.ToUpper(jsNoExtend)) + } + + sd3 := createDir(t, JetStreamStoreDir) + defer os.RemoveAll(sd3) + // deliberately pick server sA and proxy + confLA := createConfFile(t, []byte(fmt.Sprintf(tmplLA, sd3, hint, sA.opts.LeafNode.Port, lPort))) + defer removeFile(t, confLA) + sLA, _ := RunServerWithConfig(confLA) + defer sLA.Shutdown() + + sd4 := createDir(t, JetStreamStoreDir) + defer os.RemoveAll(sd4) + // deliberately pick server sA and proxy + confLB := createConfFile(t, []byte(fmt.Sprintf(tmplLB, sd4, hint, sA.opts.LeafNode.Port, lPort))) + defer removeFile(t, confLB) + sLB, _ := RunServerWithConfig(confLB) + defer sLB.Shutdown() + + checkClusterFormed(t, sLA, sLB) + + strmCfg := func(name, placementCluster string) *nats.StreamConfig { + if placementCluster == "" { + return &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}} + } + return &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}, + Placement: &nats.Placement{Cluster: placementCluster}} + } + // Only after the system account is fully connected can streams be placed anywhere. + testJSFunctions := func(pass bool) { + ncA := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sA.opts.Port)) + defer ncA.Close() + jsA, err := ncA.JetStream() + require_NoError(t, err) + _, err = jsA.AddStream(strmCfg(fmt.Sprintf("fooA1-%t", pass), "")) + require_NoError(t, err) + _, err = jsA.AddStream(strmCfg(fmt.Sprintf("fooA2-%t", pass), "clust1")) + require_NoError(t, err) + _, err = jsA.AddStream(strmCfg(fmt.Sprintf("fooA3-%t", pass), "clustL")) + if pass { + require_NoError(t, err) + } else { + require_Error(t, err) + require_Contains(t, err.Error(), "insufficient resources") + } + ncL := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sLA.opts.Port)) + defer ncL.Close() + jsL, err := ncL.JetStream() + require_NoError(t, err) + _, err = jsL.AddStream(strmCfg(fmt.Sprintf("fooL1-%t", pass), "")) + require_NoError(t, err) + _, err = jsL.AddStream(strmCfg(fmt.Sprintf("fooL2-%t", pass), "clustL")) + require_NoError(t, err) + _, err = jsL.AddStream(strmCfg(fmt.Sprintf("fooL3-%t", pass), "clust1")) + if pass { + require_NoError(t, err) + } else { + require_Error(t, err) + require_Contains(t, err.Error(), "insufficient resources") + } + } + clusterLnCnt := func(expected int) error { + cnt := 0 + for _, s := range c.servers { + cnt += s.NumLeafNodes() + } + if cnt == expected { + return nil + } + return fmt.Errorf("not enought leaf node connections, got %d needed %d", cnt, expected) + } + + // Even though there are two remotes defined in tmplL, only one will be able to connect. + checkFor(t, 10*time.Second, time.Second/4, func() error { return clusterLnCnt(2) }) + checkLeafNodeConnectedCount(t, sLA, 1) + checkLeafNodeConnectedCount(t, sLB, 1) + c.waitOnPeerCount(2) + + if testCase.leafFunctionPreJoin { + cl := cluster{t: t, servers: []*Server{sLA, sLB}} + cl.waitOnLeader() + cl.waitOnPeerCount(2) + testJSFunctions(false) + } else { + // In cases where the leaf nodes have to wait for the system account to connect, + // JetStream should not be operational during that time + ncA := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sLA.opts.Port)) + defer ncA.Close() + jsA, err := ncA.JetStream() + require_NoError(t, err) + _, err = jsA.AddStream(strmCfg("fail-false", "")) + require_Error(t, err) + } + // Starting the proxy will connect the system accounts. + // After they are connected the clusters are merged. + // Once this happened, all streams in test can be placed anywhere in the cluster. + // Before that only the cluster the client is connected to can be used for placement + p.start() + + // Even though there are two remotes defined in tmplL, only one will be able to connect. + checkFor(t, 10*time.Second, time.Second/4, func() error { return clusterLnCnt(4) }) + checkLeafNodeConnectedCount(t, sLA, 2) + checkLeafNodeConnectedCount(t, sLB, 2) + + // The leader will reside in the main cluster only + c.waitOnPeerCount(4) + testJSFunctions(true) + }) + } +} + +func TestJetStreamLeafNodeClusterMixedModeExtensionWithSystemAccount(t *testing.T) { + /* Topology used in this test: + CLUSTER(A <-> B <-> C (NO JS)) + ^ + | + LA + */ + + // once every server is up, we expect these peers to be part of the JetStream meta cluster + expectedJetStreamPeers := map[string]struct{}{ + "A": {}, + "B": {}, + "LA": {}, + } + + tmplA := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account: SYS +leafnodes: { + listen: 127.0.0.1:-1 + no_advertise: true + authorization: { + timeout: 0.5 + } +} +jetstream: { %s store_dir: '%s'; max_mem: 50Mb, max_file: 50Mb } +server_name: A +cluster: { + name: clust1 + listen: 127.0.0.1:50554 + routes=[nats-route://127.0.0.1:50555,nats-route://127.0.0.1:50556] + no_advertise: true +} +` + + tmplB := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account: SYS +leafnodes: { + listen: 127.0.0.1:-1 + no_advertise: true + authorization: { + timeout: 0.5 + } +} +jetstream: { %s store_dir: '%s'; max_mem: 50Mb, max_file: 50Mb } +server_name: B +cluster: { + name: clust1 + listen: 127.0.0.1:50555 + routes=[nats-route://127.0.0.1:50554,nats-route://127.0.0.1:50556] + no_advertise: true +} +` + + tmplC := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account: SYS +leafnodes: { + listen: 127.0.0.1:-1 + no_advertise: true + authorization: { + timeout: 0.5 + } +} +jetstream: { + enabled: false + %s +} +server_name: C +cluster: { + name: clust1 + listen: 127.0.0.1:50556 + routes=[nats-route://127.0.0.1:50554,nats-route://127.0.0.1:50555] + no_advertise: true +} +` + + tmplLA := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account = SYS +# the extension hint is to simplify this test. without it present we would need a cluster of size 2 +jetstream: { %s store_dir: '%s'; max_mem: 50Mb, max_file: 50Mb, extension_hint: will_extend } +server_name: LA +leafnodes:{ + no_advertise: true + remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A}, + {url:nats://s1:s1@127.0.0.1:%d, account: SYS}] +} +# add the cluster here so we can test placement +cluster: { name: clustL } +` + for _, withDomain := range []bool{true, false} { + t.Run(fmt.Sprintf("with-domain:%t", withDomain), func(t *testing.T) { + jsDisabledDomainString := _EMPTY_ + jsEnabledDomainString := _EMPTY_ + if withDomain { + jsEnabledDomainString = `domain: "domain", ` + jsDisabledDomainString = `domain: "domain"` + } else { + // in case no domain name is set, fall back to the extension hint. + // since JS is disabled, the value of this does not clash with other uses. + jsDisabledDomainString = "extension_hint: will_extend" + } + + sd1 := createDir(t, JetStreamStoreDir) + defer os.RemoveAll(sd1) + confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, jsEnabledDomainString, sd1))) + defer removeFile(t, confA) + sA, _ := RunServerWithConfig(confA) + defer sA.Shutdown() + + sd2 := createDir(t, JetStreamStoreDir) + defer os.RemoveAll(sd2) + confB := createConfFile(t, []byte(fmt.Sprintf(tmplB, jsEnabledDomainString, sd2))) + defer removeFile(t, confB) + sB, _ := RunServerWithConfig(confB) + defer sB.Shutdown() + + confC := createConfFile(t, []byte(fmt.Sprintf(tmplC, jsDisabledDomainString))) + defer removeFile(t, confC) + sC, _ := RunServerWithConfig(confC) + defer sC.Shutdown() + + checkClusterFormed(t, sA, sB, sC) + c := cluster{t: t, servers: []*Server{sA, sB, sC}} + c.waitOnPeerCount(2) + + sd3 := createDir(t, JetStreamStoreDir) + defer os.RemoveAll(sd3) + // deliberately pick server sC (no JS) to connect to + confLA := createConfFile(t, []byte(fmt.Sprintf(tmplLA, jsEnabledDomainString, sd3, sC.opts.LeafNode.Port, sC.opts.LeafNode.Port))) + defer removeFile(t, confLA) + sLA, _ := RunServerWithConfig(confLA) + defer sLA.Shutdown() + + checkLeafNodeConnectedCount(t, sC, 2) + checkLeafNodeConnectedCount(t, sLA, 2) + c.waitOnPeerCount(3) + peers := c.leader().JetStreamClusterPeers() + for _, peer := range peers { + if _, ok := expectedJetStreamPeers[peer]; !ok { + t.Fatalf("Found unexpected peer %q", peer) + } + } + + // helper to create stream config with uniqe name and subject + cnt := 0 + strmCfg := func(placementCluster string) *nats.StreamConfig { + name := fmt.Sprintf("s-%d", cnt) + cnt++ + if placementCluster == "" { + return &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}} + } + return &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}, + Placement: &nats.Placement{Cluster: placementCluster}} + } + + test := func(port int, expectedDefPlacement string) { + ncA := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", port)) + defer ncA.Close() + jsA, err := ncA.JetStream() + require_NoError(t, err) + si, err := jsA.AddStream(strmCfg("")) + require_NoError(t, err) + require_Contains(t, si.Cluster.Name, expectedDefPlacement) + si, err = jsA.AddStream(strmCfg("clust1")) + require_NoError(t, err) + require_Contains(t, si.Cluster.Name, "clust1") + si, err = jsA.AddStream(strmCfg("clustL")) + require_NoError(t, err) + require_Contains(t, si.Cluster.Name, "clustL") + } + + test(sA.opts.Port, "clust1") + test(sB.opts.Port, "clust1") + test(sC.opts.Port, "clust1") + test(sLA.opts.Port, "clustL") + }) + } +} + +func TestJetStreamLeafNodeCredsDenies(t *testing.T) { + tmplL := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account = SYS +jetstream: { + domain: "cluster" + store_dir: '%s' + max_mem: 50Mb + max_file: 50Mb +} +leafnodes:{ + remotes:[{url:nats://a1:a1@127.0.0.1:50555, account: A, credentials: '%s' }, + {url:nats://s1:s1@127.0.0.1:50555, account: SYS, credentials: '%s', deny_imports: foo, deny_exports: bar}] +} +` + akp, err := nkeys.CreateAccount() + require_NoError(t, err) + creds := createUserWithLimit(t, akp, time.Time{}, func(pl *jwt.UserPermissionLimits) { + pl.Pub.Deny.Add(jsAllAPI) + pl.Sub.Deny.Add(jsAllAPI) + }) + + sd := createDir(t, JetStreamStoreDir) + defer os.RemoveAll(sd) + + confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, sd, creds, creds))) + defer removeFile(t, confL) + opts := LoadConfig(confL) + sL, err := NewServer(opts) + require_NoError(t, err) + + l := captureNoticeLogger{} + sL.SetLogger(&l, false, false) + + go sL.Start() + defer sL.Shutdown() + + // wait till the notices got printed +UNTIL_READY: + for { + <-time.After(50 * time.Millisecond) + l.Lock() + for _, n := range l.notices { + if strings.Contains(n, "Server is ready") { + l.Unlock() + break UNTIL_READY + } + } + l.Unlock() + } + + l.Lock() + cnt := 0 + for _, n := range l.notices { + if strings.Contains(n, "LeafNode Remote for Account A uses credentials file") || + strings.Contains(n, "LeafNode Remote for System Account uses") || + strings.Contains(n, "Remote for System Account uses restricted export permissions") || + strings.Contains(n, "Remote for System Account uses restricted import permissions") { + cnt++ + } + } + l.Unlock() + require_True(t, cnt == 4) +} + +func TestJetStreamLeafNodeDefaultDomainCfg(t *testing.T) { + tmplHub := ` +listen: 127.0.0.1:%d +accounts :{ + A:{ jetstream: %s, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account: SYS +jetstream : %s +server_name: HUB +leafnodes: { + listen: 127.0.0.1:%d +} +%s +` + + tmplL := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account: SYS +jetstream: { domain: "%s", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } +server_name: LEAF +leafnodes: { + remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},%s] +} +%s +` + + test := func(domain string, sysShared bool) { + confHub := createConfFile(t, []byte(fmt.Sprintf(tmplHub, -1, "disabled", "disabled", -1, ""))) + defer removeFile(t, confHub) + sHub, _ := RunServerWithConfig(confHub) + defer sHub.Shutdown() + + noDomainFix := "" + if domain == _EMPTY_ { + noDomainFix = `default_js_domain:{A:""}` + } + + sys := "" + if sysShared { + sys = fmt.Sprintf(`{url:nats://s1:s1@127.0.0.1:%d, account: SYS}`, sHub.opts.LeafNode.Port) + } + + sdLeaf := createDir(t, JetStreamStoreDir) + defer os.RemoveAll(sdLeaf) + confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, domain, sdLeaf, sHub.opts.LeafNode.Port, sys, noDomainFix))) + defer removeFile(t, confL) + sLeaf, _ := RunServerWithConfig(confL) + defer sLeaf.Shutdown() + + lnCnt := 1 + if sysShared { + lnCnt++ + } + + checkLeafNodeConnectedCount(t, sHub, lnCnt) + checkLeafNodeConnectedCount(t, sLeaf, lnCnt) + + ncA := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sHub.opts.Port)) + defer ncA.Close() + jsA, err := ncA.JetStream() + require_NoError(t, err) + + _, err = jsA.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, Subjects: []string{"foo"}}) + require_True(t, err == nats.ErrNoResponders) + + // Add in default domain and restart server + require_NoError(t, ioutil.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub, + sHub.opts.Port, + "disabled", + "disabled", + sHub.opts.LeafNode.Port, + fmt.Sprintf(`default_js_domain: {A:"%s"}`, domain))), 0664)) + + sHub.Shutdown() + sHub.WaitForShutdown() + checkLeafNodeConnectedCount(t, sLeaf, 0) + sHubUpd1, _ := RunServerWithConfig(confHub) + defer sHubUpd1.Shutdown() + + checkLeafNodeConnectedCount(t, sHubUpd1, lnCnt) + checkLeafNodeConnectedCount(t, sLeaf, lnCnt) + + _, err = jsA.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, Subjects: []string{"foo"}}) + require_NoError(t, err) + + // Enable jetstream in hub. + sdHub := createDir(t, JetStreamStoreDir) + defer os.RemoveAll(sdHub) + jsEnabled := fmt.Sprintf(`{ domain: "%s", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb }`, domain, sdHub) + require_NoError(t, ioutil.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub, + sHubUpd1.opts.Port, + "disabled", + jsEnabled, + sHubUpd1.opts.LeafNode.Port, + fmt.Sprintf(`default_js_domain: {A:"%s"}`, domain))), 0664)) + + sHubUpd1.Shutdown() + sHubUpd1.WaitForShutdown() + checkLeafNodeConnectedCount(t, sLeaf, 0) + sHubUpd2, _ := RunServerWithConfig(confHub) + defer sHubUpd2.Shutdown() + + checkLeafNodeConnectedCount(t, sHubUpd2, lnCnt) + checkLeafNodeConnectedCount(t, sLeaf, lnCnt) + + _, err = jsA.AddStream(&nats.StreamConfig{Name: "bar", Replicas: 1, Subjects: []string{"bar"}}) + require_NoError(t, err) + + // Enable jetstream in account A of hub + // This is a mis config, as you can't have it both ways, local jetstream but default to another one + require_NoError(t, ioutil.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub, + sHubUpd2.opts.Port, + "enabled", + jsEnabled, + sHubUpd2.opts.LeafNode.Port, + fmt.Sprintf(`default_js_domain: {A:"%s"}`, domain))), 0664)) + + if domain != _EMPTY_ { + // in case no domain name exists there are no additional guard rails, hence no error + // It is the users responsibility to get this edge case right + sHubUpd2.Shutdown() + sHubUpd2.WaitForShutdown() + checkLeafNodeConnectedCount(t, sLeaf, 0) + sHubUpd3, err := NewServer(LoadConfig(confHub)) + sHubUpd3.Shutdown() + + require_Error(t, err) + require_Contains(t, err.Error(), `default_js_domain contains account name "A" with enabled JetStream`) + } + } + + t.Run("with-domain-sys", func(t *testing.T) { + test("domain", true) + }) + t.Run("with-domain-nosys", func(t *testing.T) { + test("domain", false) + }) + t.Run("no-domain", func(t *testing.T) { + test("", true) + }) + t.Run("no-domain", func(t *testing.T) { + test("", false) + }) +} + +func TestJetStreamLeafNodeDefaultDomainJwtExplicit(t *testing.T) { + tmplHub := ` +listen: 127.0.0.1:%d +operator: %s +system_account: %s +resolver: MEM +resolver_preload: { + %s:%s + %s:%s +} +jetstream : disabled +server_name: HUB +leafnodes: { + listen: 127.0.0.1:%d +} +%s +` + + tmplL := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, + SYS:{ users:[ {user:s1,password:s1}]}, +} +system_account: SYS +jetstream: { domain: "%s", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } +server_name: LEAF +leafnodes: { + remotes:[{url:nats://127.0.0.1:%d, account: A, credentials: '%s'}, + {url:nats://127.0.0.1:%d, account: SYS, credentials: '%s'}] +} +%s +` + + test := func(domain string) { + noDomainFix := "" + if domain == _EMPTY_ { + noDomainFix = `default_js_domain:{A:""}` + } + + sysKp, syspub := createKey(t) + sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) + sysCreds := newUser(t, sysKp) + defer removeFile(t, sysCreds) + + aKp, aPub := createKey(t) + aClaim := jwt.NewAccountClaims(aPub) + aJwt := encodeClaim(t, aClaim, aPub) + aCreds := newUser(t, aKp) + defer removeFile(t, aCreds) + + confHub := createConfFile(t, []byte(fmt.Sprintf(tmplHub, -1, ojwt, syspub, syspub, sysJwt, aPub, aJwt, -1, ""))) + defer removeFile(t, confHub) + sHub, _ := RunServerWithConfig(confHub) + defer sHub.Shutdown() + + sdLeaf := createDir(t, JetStreamStoreDir) + defer os.RemoveAll(sdLeaf) + confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, + domain, + sdLeaf, + sHub.opts.LeafNode.Port, + aCreds, + sHub.opts.LeafNode.Port, + sysCreds, + noDomainFix))) + defer removeFile(t, confL) + sLeaf, _ := RunServerWithConfig(confL) + defer sLeaf.Shutdown() + + checkLeafNodeConnectedCount(t, sHub, 2) + checkLeafNodeConnectedCount(t, sLeaf, 2) + + ncA := natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", sHub.opts.Port), createUserCreds(t, nil, aKp)) + defer ncA.Close() + jsA, err := ncA.JetStream() + require_NoError(t, err) + + _, err = jsA.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, Subjects: []string{"foo"}}) + require_True(t, err == nats.ErrNoResponders) + + // Add in default domain and restart server + require_NoError(t, ioutil.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub, + sHub.opts.Port, ojwt, syspub, syspub, sysJwt, aPub, aJwt, sHub.opts.LeafNode.Port, + fmt.Sprintf(`default_js_domain: {%s:"%s"}`, aPub, domain))), 0664)) + + sHub.Shutdown() + sHub.WaitForShutdown() + checkLeafNodeConnectedCount(t, sLeaf, 0) + sHubUpd1, _ := RunServerWithConfig(confHub) + defer sHubUpd1.Shutdown() + + checkLeafNodeConnectedCount(t, sHubUpd1, 2) + checkLeafNodeConnectedCount(t, sLeaf, 2) + + _, err = jsA.AddStream(&nats.StreamConfig{Name: "bar", Replicas: 1, Subjects: []string{"bar"}}) + require_NoError(t, err) + } + t.Run("with-domain", func(t *testing.T) { + test("domain") + }) + t.Run("no-domain", func(t *testing.T) { + test("") + }) +} + +func TestJetStreamLeafNodeDefaultDomainClusterBothEnds(t *testing.T) { + // test to ensure that default domain functions when both ends of the leaf node connection are clusters + tmplHub1 := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enabled, users:[ {user:a1,password:a1}]}, + B:{ jetstream: enabled, users:[ {user:b1,password:b1}]} +} +jetstream : { domain: "DHUB", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } +server_name: HUB1 +cluster: { + name: HUB + listen: 127.0.0.1:50554 + routes=[nats-route://127.0.0.1:50555] +} +leafnodes: { + listen:127.0.0.1:-1 +} +` + + tmplHub2 := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enabled, users:[ {user:a1,password:a1}]}, + B:{ jetstream: enabled, users:[ {user:b1,password:b1}]} +} +jetstream : { domain: "DHUB", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } +server_name: HUB2 +cluster: { + name: HUB + listen: 127.0.0.1:50555 + routes=[nats-route://127.0.0.1:50554] +} +leafnodes: { + listen:127.0.0.1:-1 +} +` + + tmplL1 := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enabled, users:[ {user:a1,password:a1}]}, + B:{ jetstream: disabled, users:[ {user:b1,password:b1}]} +} +jetstream: { domain: "DLEAF", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } +server_name: LEAF1 +cluster: { + name: LEAF + listen: 127.0.0.1:50556 + routes=[nats-route://127.0.0.1:50557] +} +leafnodes: { + remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},{url:nats://b1:b1@127.0.0.1:%d, account: B}] +} +default_js_domain: {B:"DHUB"} +` + + tmplL2 := ` +listen: 127.0.0.1:-1 +accounts :{ + A:{ jetstream: enabled, users:[ {user:a1,password:a1}]}, + B:{ jetstream: disabled, users:[ {user:b1,password:b1}]} +} +jetstream: { domain: "DLEAF", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } +server_name: LEAF2 +cluster: { + name: LEAF + listen: 127.0.0.1:50557 + routes=[nats-route://127.0.0.1:50556] +} +leafnodes: { + remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},{url:nats://b1:b1@127.0.0.1:%d, account: B}] +} +default_js_domain: {B:"DHUB"} +` + + sd1 := createDir(t, JetStreamStoreDir) + defer os.RemoveAll(sd1) + confHub1 := createConfFile(t, []byte(fmt.Sprintf(tmplHub1, sd1))) + defer removeFile(t, confHub1) + sHub1, _ := RunServerWithConfig(confHub1) + defer sHub1.Shutdown() + + sd2 := createDir(t, JetStreamStoreDir) + defer os.RemoveAll(sd2) + confHub2 := createConfFile(t, []byte(fmt.Sprintf(tmplHub2, sd2))) + defer removeFile(t, confHub2) + sHub2, _ := RunServerWithConfig(confHub2) + defer sHub2.Shutdown() + + checkClusterFormed(t, sHub1, sHub2) + c1 := cluster{t: t, servers: []*Server{sHub1, sHub2}} + c1.waitOnPeerCount(2) + + sd3 := createDir(t, JetStreamStoreDir) + defer os.RemoveAll(sd3) + confLeaf1 := createConfFile(t, []byte(fmt.Sprintf(tmplL1, sd3, sHub1.getOpts().LeafNode.Port, sHub1.getOpts().LeafNode.Port))) + defer removeFile(t, confLeaf1) + sLeaf1, _ := RunServerWithConfig(confLeaf1) + defer sLeaf1.Shutdown() + + sd4 := createDir(t, JetStreamStoreDir) + defer os.RemoveAll(sd4) + confLeaf2 := createConfFile(t, []byte(fmt.Sprintf(tmplL2, sd3, sHub1.getOpts().LeafNode.Port, sHub1.getOpts().LeafNode.Port))) + defer removeFile(t, confLeaf2) + sLeaf2, _ := RunServerWithConfig(confLeaf2) + defer sLeaf2.Shutdown() + + checkClusterFormed(t, sLeaf1, sLeaf2) + c2 := cluster{t: t, servers: []*Server{sLeaf1, sLeaf2}} + c2.waitOnPeerCount(2) + + checkLeafNodeConnectedCount(t, sHub1, 4) + checkLeafNodeConnectedCount(t, sLeaf1, 2) + checkLeafNodeConnectedCount(t, sLeaf2, 2) + + ncB := natsConnect(t, fmt.Sprintf("nats://b1:b1@127.0.0.1:%d", sLeaf1.getOpts().Port)) + defer ncB.Close() + jsB1, err := ncB.JetStream() + require_NoError(t, err) + si, err := jsB1.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, Subjects: []string{"foo"}}) + require_NoError(t, err) + require_Equal(t, si.Cluster.Name, "HUB") + + jsB2, err := ncB.JetStream(nats.Domain("DHUB")) + require_NoError(t, err) + si, err = jsB2.AddStream(&nats.StreamConfig{Name: "bar", Replicas: 1, Subjects: []string{"bar"}}) + require_NoError(t, err) + require_Equal(t, si.Cluster.Name, "HUB") +} diff --git a/server/jetstream_test.go b/server/jetstream_test.go index 4d6c8aa5..2f9327c3 100644 --- a/server/jetstream_test.go +++ b/server/jetstream_test.go @@ -11,6 +11,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !skip_js_tests +// +build !skip_js_tests + package server import ( @@ -317,7 +320,7 @@ func TestJetStreamAutoTuneFSConfig(t *testing.T) { s := RunRandClientPortServer() defer s.Shutdown() - jsconfig := &JetStreamConfig{MaxMemory: -1, MaxStore: 128 * 1024 * 1024 * 1024} + jsconfig := &JetStreamConfig{MaxMemory: -1, MaxStore: 128 * 1024 * 1024} if err := s.EnableJetStream(jsconfig); err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -361,7 +364,7 @@ func TestJetStreamAutoTuneFSConfig(t *testing.T) { testBlkSize("foo", 1, 512, FileStoreMinBlkSize) testBlkSize("foo", 1, 1024*1024, defaultMediumBlockSize) testBlkSize("foo", 1, 8*1024*1024, defaultMediumBlockSize) - testBlkSize("foo_bar_baz", -1, 32*1024*1024*1024, FileStoreMaxBlkSize) + testBlkSize("foo_bar_baz", -1, 32*1024*1024, FileStoreMaxBlkSize) } func TestJetStreamConsumerAndStreamDescriptions(t *testing.T) { diff --git a/server/jwt_test.go b/server/jwt_test.go index df1a8644..a148ecf1 100644 --- a/server/jwt_test.go +++ b/server/jwt_test.go @@ -4196,185 +4196,6 @@ func TestJWTNoOperatorMode(t *testing.T) { } } -func TestJWTJetStreamLimits(t *testing.T) { - updateJwt := func(url string, creds string, pubKey string, jwt string) { - t.Helper() - c := natsConnect(t, url, nats.UserCredentials(creds)) - defer c.Close() - if msg, err := c.Request(fmt.Sprintf(accUpdateEventSubjNew, pubKey), []byte(jwt), time.Second); err != nil { - t.Fatal("error not expected in this test", err) - } else { - content := make(map[string]interface{}) - if err := json.Unmarshal(msg.Data, &content); err != nil { - t.Fatalf("%v", err) - } else if _, ok := content["data"]; !ok { - t.Fatalf("did not get an ok response got: %v", content) - } - } - } - require_IdenticalLimits := func(infoLim JetStreamAccountLimits, lim jwt.JetStreamLimits) { - t.Helper() - if int64(infoLim.MaxConsumers) != lim.Consumer || int64(infoLim.MaxStreams) != lim.Streams || - infoLim.MaxMemory != lim.MemoryStorage || infoLim.MaxStore != lim.DiskStorage { - t.Fatalf("limits do not match %v != %v", infoLim, lim) - } - } - expect_JSDisabledForAccount := func(c *nats.Conn) { - t.Helper() - if _, err := c.Request("$JS.API.INFO", nil, time.Second); err != nats.ErrTimeout && err != nats.ErrNoResponders { - t.Fatalf("Unexpected error: %v", err) - } - } - expect_InfoError := func(c *nats.Conn) { - t.Helper() - var info JSApiAccountInfoResponse - if resp, err := c.Request("$JS.API.INFO", nil, time.Second); err != nil { - t.Fatalf("Unexpected error: %v", err) - } else if err = json.Unmarshal(resp.Data, &info); err != nil { - t.Fatalf("response1 %v got error %v", string(resp.Data), err) - } else if info.Error == nil { - t.Fatalf("expected error") - } - } - validate_limits := func(c *nats.Conn, expectedLimits jwt.JetStreamLimits) { - t.Helper() - var info JSApiAccountInfoResponse - if resp, err := c.Request("$JS.API.INFO", nil, time.Second); err != nil { - t.Fatalf("Unexpected error: %v", err) - } else if err = json.Unmarshal(resp.Data, &info); err != nil { - t.Fatalf("response1 %v got error %v", string(resp.Data), err) - } else { - require_IdenticalLimits(info.Limits, expectedLimits) - } - } - // create system account - sysKp, _ := nkeys.CreateAccount() - sysPub, _ := sysKp.PublicKey() - sysUKp, _ := nkeys.CreateUser() - sysUSeed, _ := sysUKp.Seed() - uclaim := newJWTTestUserClaims() - uclaim.Subject, _ = sysUKp.PublicKey() - sysUserJwt, err := uclaim.Encode(sysKp) - require_NoError(t, err) - sysKp.Seed() - sysCreds := genCredsFile(t, sysUserJwt, sysUSeed) - // limits to apply and check - limits1 := jwt.JetStreamLimits{MemoryStorage: 1024 * 1024, DiskStorage: 2048 * 1024, Streams: 1, Consumer: 2, MaxBytesRequired: true} - // has valid limits that would fail when incorrectly applied twice - limits2 := jwt.JetStreamLimits{MemoryStorage: 4096 * 1024, DiskStorage: 8192 * 1024, Streams: 3, Consumer: 4} - // limits exceeding actual configured value of DiskStorage - limitsExceeded := jwt.JetStreamLimits{MemoryStorage: 8192 * 1024, DiskStorage: 16384 * 1024, Streams: 5, Consumer: 6} - // create account using jetstream with both limits - akp, _ := nkeys.CreateAccount() - aPub, _ := akp.PublicKey() - claim := jwt.NewAccountClaims(aPub) - claim.Limits.JetStreamLimits = limits1 - aJwt1, err := claim.Encode(oKp) - require_NoError(t, err) - claim.Limits.JetStreamLimits = limits2 - aJwt2, err := claim.Encode(oKp) - require_NoError(t, err) - claim.Limits.JetStreamLimits = limitsExceeded - aJwtLimitsExceeded, err := claim.Encode(oKp) - require_NoError(t, err) - claim.Limits.JetStreamLimits = jwt.JetStreamLimits{} // disabled - aJwt4, err := claim.Encode(oKp) - require_NoError(t, err) - // account user - uKp, _ := nkeys.CreateUser() - uSeed, _ := uKp.Seed() - uclaim = newJWTTestUserClaims() - uclaim.Subject, _ = uKp.PublicKey() - userJwt, err := uclaim.Encode(akp) - require_NoError(t, err) - userCreds := genCredsFile(t, userJwt, uSeed) - dir := createDir(t, "srv") - defer removeDir(t, dir) - conf := createConfFile(t, []byte(fmt.Sprintf(` - listen: 127.0.0.1:-1 - jetstream: {max_mem_store: 10Mb, max_file_store: 10Mb} - operator: %s - resolver: { - type: full - dir: '%s' - } - system_account: %s - `, ojwt, dir, sysPub))) - defer removeFile(t, conf) - s, opts := RunServerWithConfig(conf) - defer s.Shutdown() - port := opts.Port - updateJwt(s.ClientURL(), sysCreds, aPub, aJwt1) - c := natsConnect(t, s.ClientURL(), nats.UserCredentials(userCreds), nats.ReconnectWait(200*time.Millisecond)) - defer c.Close() - validate_limits(c, limits1) - // keep using the same connection - updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) - validate_limits(c, limits2) - // keep using the same connection but do NOT CHANGE anything. - // This tests if the jwt is applied a second time (would fail) - updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) - validate_limits(c, limits2) - // keep using the same connection. This update EXCEEDS LIMITS - updateJwt(s.ClientURL(), sysCreds, aPub, aJwtLimitsExceeded) - validate_limits(c, limits2) - // disable test after failure - updateJwt(s.ClientURL(), sysCreds, aPub, aJwt4) - expect_InfoError(c) - // re enable, again testing with a value that can't be applied twice - updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) - validate_limits(c, limits2) - // disable test no prior failure - updateJwt(s.ClientURL(), sysCreds, aPub, aJwt4) - expect_InfoError(c) - // Wrong limits form start - updateJwt(s.ClientURL(), sysCreds, aPub, aJwtLimitsExceeded) - expect_JSDisabledForAccount(c) - // enable js but exceed limits. Followed by fix via restart - updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) - validate_limits(c, limits2) - updateJwt(s.ClientURL(), sysCreds, aPub, aJwtLimitsExceeded) - validate_limits(c, limits2) - s.Shutdown() - conf = createConfFile(t, []byte(fmt.Sprintf(` - listen: 127.0.0.1:%d - jetstream: {max_mem_store: 20Mb, max_file_store: 20Mb} - operator: %s - resolver: { - type: full - dir: '%s' - } - system_account: %s - `, port, ojwt, dir, sysPub))) - defer removeFile(t, conf) - s, _ = RunServerWithConfig(conf) - defer s.Shutdown() - c.Flush() // force client to discover the disconnect - checkClientsCount(t, s, 1) - validate_limits(c, limitsExceeded) - s.Shutdown() - // disable jetstream test - conf = createConfFile(t, []byte(fmt.Sprintf(` - listen: 127.0.0.1:%d - operator: %s - resolver: { - type: full - dir: '%s' - } - system_account: %s - `, port, ojwt, dir, sysPub))) - defer removeFile(t, conf) - s, _ = RunServerWithConfig(conf) - defer s.Shutdown() - c.Flush() // force client to discover the disconnect - checkClientsCount(t, s, 1) - expect_JSDisabledForAccount(c) - // test that it stays disabled - updateJwt(s.ClientURL(), sysCreds, aPub, aJwt2) - expect_JSDisabledForAccount(c) - c.Close() -} - func TestJWTUserRevocation(t *testing.T) { test := func(all bool) { createAccountAndUser := func(done chan struct{}, pubKey, jwt1, jwt2, creds1, creds2 *string) { @@ -5590,429 +5411,6 @@ func TestJWTJetStreamMaxStreamBytes(t *testing.T) { require_Equal(t, err.Error(), "account requires a stream config to have max bytes set") } -func TestJWTJetStreamMoveWithTiers(t *testing.T) { - _, syspub := createKey(t) - sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) - - accKp, aExpPub := createKey(t) - accClaim := jwt.NewAccountClaims(aExpPub) - accClaim.Name = "acc" - accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ - DiskStorage: 1100, Consumer: 1, Streams: 1} - accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ - DiskStorage: 3300, Consumer: 1, Streams: 1} - accJwt := encodeClaim(t, accClaim, aExpPub) - accCreds := newUser(t, accKp) - - test := func(t *testing.T, replicas int) { - tmlp := ` - listen: 127.0.0.1:-1 - server_name: %s - jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} - leaf { - listen: 127.0.0.1:-1 - } - cluster { - name: %s - listen: 127.0.0.1:%d - routes = [%s] - } - ` - s := createJetStreamSuperClusterWithTemplateAndModHook(t, tmlp, 3, 3, - func(serverName, clustername, storeDir, conf string) string { - return conf + fmt.Sprintf(` - server_tags: [cloud:%s-tag] - operator: %s - system_account: %s - resolver = MEMORY - resolver_preload = { - %s : %s - %s : %s - } - `, clustername, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt) - }) - defer s.shutdown() - - nc := natsConnect(t, s.randomServer().ClientURL(), nats.UserCredentials(accCreds)) - defer nc.Close() - - js, err := nc.JetStream() - require_NoError(t, err) - - ci, err := js.AddStream(&nats.StreamConfig{Name: "MOVE-ME", Replicas: replicas, - Placement: &nats.Placement{Tags: []string{"cloud:C1-tag"}}}) - require_NoError(t, err) - require_Equal(t, ci.Cluster.Name, "C1") - ci, err = js.UpdateStream(&nats.StreamConfig{Name: "MOVE-ME", Replicas: replicas, - Placement: &nats.Placement{Tags: []string{"cloud:C2-tag"}}}) - require_NoError(t, err) - require_Equal(t, ci.Cluster.Name, "C1") - - checkFor(t, 10*time.Second, 10*time.Millisecond, func() error { - if si, err := js.StreamInfo("MOVE-ME"); err != nil { - return err - } else if si.Cluster.Name != "C2" { - return fmt.Errorf("Wrong cluster: %q", si.Cluster.Name) - } else if si.Cluster.Leader == _EMPTY_ { - return fmt.Errorf("No leader yet") - } else if !strings.HasPrefix(si.Cluster.Leader, "C2-") { - return fmt.Errorf("Wrong leader: %q", si.Cluster.Leader) - } else if len(si.Cluster.Replicas) != replicas-1 { - return fmt.Errorf("Expected %d replicas, got %d", replicas-1, len(si.Cluster.Replicas)) - } - return nil - }) - } - - t.Run("R1", func(t *testing.T) { - test(t, 1) - }) - t.Run("R3", func(t *testing.T) { - test(t, 3) - }) -} - -func TestJWTClusteredJetStreamTiers(t *testing.T) { - sysKp, syspub := createKey(t) - sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) - sysCreds := newUser(t, sysKp) - defer removeFile(t, sysCreds) - - accKp, aExpPub := createKey(t) - accClaim := jwt.NewAccountClaims(aExpPub) - accClaim.Name = "acc" - accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ - DiskStorage: 1100, Consumer: 2, Streams: 2} - accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ - DiskStorage: 3300, Consumer: 1, Streams: 1} - accJwt := encodeClaim(t, accClaim, aExpPub) - accCreds := newUser(t, accKp) - tmlp := ` - listen: 127.0.0.1:-1 - server_name: %s - jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} - leaf { - listen: 127.0.0.1:-1 - } - cluster { - name: %s - listen: 127.0.0.1:%d - routes = [%s] - } - ` + fmt.Sprintf(` - operator: %s - system_account: %s - resolver = MEMORY - resolver_preload = { - %s : %s - %s : %s - } - `, ojwt, syspub, syspub, sysJwt, aExpPub, accJwt) - - c := createJetStreamClusterWithTemplate(t, tmlp, "cluster", 3) - defer c.shutdown() - - nc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds)) - defer nc.Close() - - js, err := nc.JetStream() - require_NoError(t, err) - - // Test absent tiers - _, err = js.AddStream(&nats.StreamConfig{Name: "testR2", Replicas: 2, Subjects: []string{"testR2"}}) - require_Error(t, err) - require_Equal(t, err.Error(), "no JetStream default or applicable tiered limit present") - _, err = js.AddStream(&nats.StreamConfig{Name: "testR5", Replicas: 5, Subjects: []string{"testR5"}}) - require_Error(t, err) - require_Equal(t, err.Error(), "no JetStream default or applicable tiered limit present") - - // Test tiers up to stream limits - _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-1", Replicas: 1, Subjects: []string{"testR1-1"}}) - require_NoError(t, err) - _, err = js.AddStream(&nats.StreamConfig{Name: "testR3-1", Replicas: 3, Subjects: []string{"testR3-1"}}) - require_NoError(t, err) - _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-2", Replicas: 1, Subjects: []string{"testR1-2"}}) - require_NoError(t, err) - - // Test exceeding tiered stream limit - _, err = js.AddStream(&nats.StreamConfig{Name: "testR1-3", Replicas: 1, Subjects: []string{"testR1-3"}}) - require_Error(t, err) - require_Equal(t, err.Error(), "maximum number of streams reached") - _, err = js.AddStream(&nats.StreamConfig{Name: "testR3-3", Replicas: 3, Subjects: []string{"testR3-3"}}) - require_Error(t, err) - require_Equal(t, err.Error(), "maximum number of streams reached") - - // Test tiers up to consumer limits - _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur1", AckPolicy: nats.AckExplicitPolicy}) - require_NoError(t, err) - _, err = js.AddConsumer("testR3-1", &nats.ConsumerConfig{Durable: "dur2", AckPolicy: nats.AckExplicitPolicy}) - require_NoError(t, err) - _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur3", AckPolicy: nats.AckExplicitPolicy}) - require_NoError(t, err) - - // test exceeding tiered consumer limits - _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur4", AckPolicy: nats.AckExplicitPolicy}) - require_Error(t, err) - require_Equal(t, err.Error(), "maximum consumers limit reached") - _, err = js.AddConsumer("testR1-1", &nats.ConsumerConfig{Durable: "dur5", AckPolicy: nats.AckExplicitPolicy}) - require_Error(t, err) - require_Equal(t, err.Error(), "maximum consumers limit reached") - - // test tiered storage limit - msg := [512]byte{} - _, err = js.Publish("testR1-1", msg[:]) - require_NoError(t, err) - _, err = js.Publish("testR3-1", msg[:]) - require_NoError(t, err) - _, err = js.Publish("testR3-1", msg[:]) - require_NoError(t, err) - _, err = js.Publish("testR1-2", msg[:]) - require_NoError(t, err) - - time.Sleep(2000 * time.Millisecond) // wait for update timer to synchronize totals - - // test exceeding tiered storage limit - _, err = js.Publish("testR1-1", []byte("1")) - require_Error(t, err) - require_Equal(t, err.Error(), "nats: resource limits exceeded for account") - _, err = js.Publish("testR3-1", []byte("fail this message!")) - require_Error(t, err) - require_Equal(t, err.Error(), "nats: resource limits exceeded for account") - - // retrieve limits - var info JSApiAccountInfoResponse - m, err := nc.Request("$JS.API.INFO", nil, time.Second) - require_NoError(t, err) - err = json.Unmarshal(m.Data, &info) - require_NoError(t, err) - - require_True(t, info.Memory == 0) - // R1 streams fail message with an add followed by remove, if the update was sent in between, the count is > limit - // Alternative to checking both values is, prior to the info request, wait for another update - require_True(t, info.Store == 4400 || info.Store == 4439) - require_True(t, info.Streams == 3) - require_True(t, info.Consumers == 3) - require_True(t, info.Limits == JetStreamAccountLimits{}) - r1 := info.Tiers["R1"] - require_True(t, r1.Streams == 2) - require_True(t, r1.Consumers == 2) - // R1 streams fail message with an add followed by remove, if the update was sent in between, the count is > limit - // Alternative to checking both values is, prior to the info request, wait for another update - require_True(t, r1.Store == 1100 || r1.Store == 1139) - require_True(t, r1.Memory == 0) - require_True(t, r1.Limits == JetStreamAccountLimits{ - MaxMemory: 0, - MaxStore: 1100, - MaxStreams: 2, - MaxConsumers: 2, - MaxAckPending: -1, - MemoryMaxStreamBytes: -1, - StoreMaxStreamBytes: -1, - MaxBytesRequired: false, - }) - r3 := info.Tiers["R3"] - require_True(t, r3.Streams == 1) - require_True(t, r3.Consumers == 1) - require_True(t, r3.Store == 3300) - require_True(t, r3.Memory == 0) - require_True(t, r3.Limits == JetStreamAccountLimits{ - MaxMemory: 0, - MaxStore: 3300, - MaxStreams: 1, - MaxConsumers: 1, - MaxAckPending: -1, - MemoryMaxStreamBytes: -1, - StoreMaxStreamBytes: -1, - MaxBytesRequired: false, - }) -} - -func TestJWTClusteredJetStreamTiersChange(t *testing.T) { - sysKp, syspub := createKey(t) - sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) - sysCreds := newUser(t, sysKp) - defer removeFile(t, sysCreds) - - accKp, aExpPub := createKey(t) - accClaim := jwt.NewAccountClaims(aExpPub) - accClaim.Name = "acc" - accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ - DiskStorage: 1000, MemoryStorage: 0, Consumer: 1, Streams: 1} - accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ - DiskStorage: 1500, MemoryStorage: 0, Consumer: 1, Streams: 1} - accJwt1 := encodeClaim(t, accClaim, aExpPub) - accCreds := newUser(t, accKp) - start := time.Now() - - tmlp := ` - listen: 127.0.0.1:-1 - server_name: %s - jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} - leaf { - listen: 127.0.0.1:-1 - } - cluster { - name: %s - listen: 127.0.0.1:%d - routes = [%s] - } - ` - c := createJetStreamClusterWithTemplateAndModHook(t, tmlp, "cluster", 3, - func(serverName, clustername, storeDir, conf string) string { - return conf + fmt.Sprintf(` - operator: %s - system_account: %s - resolver: { - type: full - dir: '%s/jwt' - }`, ojwt, syspub, storeDir) - }) - defer c.shutdown() - - updateJwt(t, c.randomServer().ClientURL(), sysCreds, sysJwt, 3) - updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt1, 3) - - nc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds)) - defer nc.Close() - - js, err := nc.JetStream() - require_NoError(t, err) - - // Test tiers up to stream limits - cfg := &nats.StreamConfig{Name: "testR1-1", Replicas: 1, Subjects: []string{"testR1-1"}, MaxBytes: 1000} - _, err = js.AddStream(cfg) - require_NoError(t, err) - - cfg.Replicas = 3 - _, err = js.UpdateStream(cfg) - require_Error(t, err) - require_Equal(t, err.Error(), "insufficient storage resources available") - - time.Sleep(time.Second - time.Since(start)) // make sure the time stamp changes - accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ - DiskStorage: 3000, MemoryStorage: 0, Consumer: 1, Streams: 1} - accJwt2 := encodeClaim(t, accClaim, aExpPub) - - updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt2, 3) - - var rBefore, rAfter JSApiAccountInfoResponse - m, err := nc.Request("$JS.API.INFO", nil, time.Second) - require_NoError(t, err) - err = json.Unmarshal(m.Data, &rBefore) - require_NoError(t, err) - _, err = js.UpdateStream(cfg) - require_NoError(t, err) - - m, err = nc.Request("$JS.API.INFO", nil, time.Second) - require_NoError(t, err) - err = json.Unmarshal(m.Data, &rAfter) - require_NoError(t, err) - require_True(t, rBefore.Tiers["R1"].Streams == 1) - require_True(t, rBefore.Tiers["R1"].Streams == rAfter.Tiers["R3"].Streams) - require_True(t, rBefore.Tiers["R3"].Streams == 0) - require_True(t, rAfter.Tiers["R1"].Streams == 0) -} - -func TestJWTClusteredJetStreamDeleteTierWithStreamAndMove(t *testing.T) { - sysKp, syspub := createKey(t) - sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) - sysCreds := newUser(t, sysKp) - defer removeFile(t, sysCreds) - - accKp, aExpPub := createKey(t) - accClaim := jwt.NewAccountClaims(aExpPub) - accClaim.Name = "acc" - accClaim.Limits.JetStreamTieredLimits["R1"] = jwt.JetStreamLimits{ - DiskStorage: 1000, MemoryStorage: 0, Consumer: 1, Streams: 1} - accClaim.Limits.JetStreamTieredLimits["R3"] = jwt.JetStreamLimits{ - DiskStorage: 3000, MemoryStorage: 0, Consumer: 1, Streams: 1} - accJwt1 := encodeClaim(t, accClaim, aExpPub) - accCreds := newUser(t, accKp) - start := time.Now() - - tmlp := ` - listen: 127.0.0.1:-1 - server_name: %s - jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} - leaf { - listen: 127.0.0.1:-1 - } - cluster { - name: %s - listen: 127.0.0.1:%d - routes = [%s] - } - ` - c := createJetStreamClusterWithTemplateAndModHook(t, tmlp, "cluster", 3, - func(serverName, clustername, storeDir, conf string) string { - return conf + fmt.Sprintf(` - operator: %s - system_account: %s - resolver: { - type: full - dir: '%s/jwt' - }`, ojwt, syspub, storeDir) - }) - defer c.shutdown() - - updateJwt(t, c.randomServer().ClientURL(), sysCreds, sysJwt, 3) - updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt1, 3) - - nc := natsConnect(t, c.randomServer().ClientURL(), nats.UserCredentials(accCreds)) - defer nc.Close() - - js, err := nc.JetStream() - require_NoError(t, err) - - // Test tiers up to stream limits - cfg := &nats.StreamConfig{Name: "testR1-1", Replicas: 1, Subjects: []string{"testR1-1"}, MaxBytes: 1000} - _, err = js.AddStream(cfg) - require_NoError(t, err) - - _, err = js.Publish("testR1-1", nil) - require_NoError(t, err) - - time.Sleep(time.Second - time.Since(start)) // make sure the time stamp changes - delete(accClaim.Limits.JetStreamTieredLimits, "R1") - accJwt2 := encodeClaim(t, accClaim, aExpPub) - updateJwt(t, c.randomServer().ClientURL(), sysCreds, accJwt2, 3) - - var respBefore JSApiAccountInfoResponse - m, err := nc.Request("$JS.API.INFO", nil, time.Second) - require_NoError(t, err) - err = json.Unmarshal(m.Data, &respBefore) - require_NoError(t, err) - - require_True(t, respBefore.JetStreamAccountStats.Tiers["R3"].Streams == 0) - require_True(t, respBefore.JetStreamAccountStats.Tiers["R1"].Streams == 1) - - _, err = js.Publish("testR1-1", nil) - require_Error(t, err) - require_Equal(t, err.Error(), "nats: no JetStream default or applicable tiered limit present") - - cfg.Replicas = 3 - _, err = js.UpdateStream(cfg) - require_NoError(t, err) - - // I noticed this taking > 5 seconds - checkFor(t, 10*time.Second, 250*time.Millisecond, func() error { - _, err = js.Publish("testR1-1", nil) - return err - }) - - var respAfter JSApiAccountInfoResponse - m, err = nc.Request("$JS.API.INFO", nil, time.Second) - require_NoError(t, err) - err = json.Unmarshal(m.Data, &respAfter) - require_NoError(t, err) - - require_True(t, respAfter.JetStreamAccountStats.Tiers["R3"].Streams == 1) - require_True(t, respAfter.JetStreamAccountStats.Tiers["R3"].Store > 0) - - _, ok := respAfter.JetStreamAccountStats.Tiers["R1"] - require_True(t, !ok) -} - func TestJWTQueuePermissions(t *testing.T) { aExpKp, aExpPub := createKey(t) aExpClaim := jwt.NewAccountClaims(aExpPub) @@ -6787,132 +6185,3 @@ func TestJWTAccountConnzAccessAfterClaimUpdate(t *testing.T) { // If export was wiped this would fail with timeout. doRequest() } - -func TestJWTSysAccUpdateMixedMode(t *testing.T) { - skp, spub := createKey(t) - sUsr := createUserCreds(t, nil, skp) - sysClaim := jwt.NewAccountClaims(spub) - sysClaim.Name = "SYS" - sysJwt := encodeClaim(t, sysClaim, spub) - encodeJwt1Time := time.Now() - - akp, apub := createKey(t) - aUsr := createUserCreds(t, nil, akp) - claim := jwt.NewAccountClaims(apub) - claim.Limits.JetStreamLimits.DiskStorage = 1024 * 1024 - claim.Limits.JetStreamLimits.Streams = 1 - jwt1 := encodeClaim(t, claim, apub) - - basePath := "/ngs/v1/accounts/jwt/" - reqCount := int32(0) - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == basePath { - w.Write([]byte("ok")) - } else if strings.HasSuffix(r.URL.Path, spub) { - w.Write([]byte(sysJwt)) - } else if strings.HasSuffix(r.URL.Path, apub) { - w.Write([]byte(jwt1)) - } else { - // only count requests that could be filled - return - } - atomic.AddInt32(&reqCount, 1) - })) - defer ts.Close() - - tmpl := ` - listen: 127.0.0.1:-1 - server_name: %s - jetstream: {max_mem_store: 256MB, max_file_store: 2GB, store_dir: '%s'} - cluster { - name: %s - listen: 127.0.0.1:%d - routes = [%s] - } - ` - - sc := createJetStreamSuperClusterWithTemplateAndModHook(t, tmpl, 3, 2, - func(serverName, clusterName, storeDir, conf string) string { - // create an ngs like setup, with connection and non connection server - if clusterName == "C1" { - conf = strings.ReplaceAll(conf, "jetstream", "#jetstream") - } - return fmt.Sprintf(`%s - operator: %s - system_account: %s - resolver: URL("%s%s")`, conf, ojwt, spub, ts.URL, basePath) - }) - defer sc.shutdown() - disconnectChan := make(chan struct{}, 100) - defer close(disconnectChan) - disconnectCb := nats.DisconnectErrHandler(func(conn *nats.Conn, err error) { - disconnectChan <- struct{}{} - }) - - s := sc.clusterForName("C1").randomServer() - - sysNc := natsConnect(t, s.ClientURL(), sUsr, disconnectCb, nats.NoCallbacksAfterClientClose()) - defer sysNc.Close() - aNc := natsConnect(t, s.ClientURL(), aUsr, disconnectCb, nats.NoCallbacksAfterClientClose()) - defer aNc.Close() - - js, err := aNc.JetStream() - require_NoError(t, err) - - si, err := js.AddStream(&nats.StreamConfig{Name: "bar", Subjects: []string{"bar"}, Replicas: 3}) - require_NoError(t, err) - require_Equal(t, si.Cluster.Name, "C2") - _, err = js.AccountInfo() - require_NoError(t, err) - - r, err := sysNc.Request(fmt.Sprintf(serverPingReqSubj, "ACCOUNTZ"), - []byte(fmt.Sprintf(`{"account":"%s"}`, spub)), time.Second) - require_NoError(t, err) - respb := ServerAPIResponse{Data: &Accountz{}} - require_NoError(t, json.Unmarshal(r.Data, &respb)) - - hasJSExp := func(resp *ServerAPIResponse) bool { - found := false - for _, e := range resp.Data.(*Accountz).Account.Exports { - if e.Subject == jsAllAPI { - found = true - break - } - } - return found - } - require_True(t, hasJSExp(&respb)) - - // make sure jti increased - time.Sleep(time.Second - time.Since(encodeJwt1Time)) - sysJwt2 := encodeClaim(t, sysClaim, spub) - - oldRcount := atomic.LoadInt32(&reqCount) - _, err = sysNc.Request(fmt.Sprintf(accUpdateEventSubjNew, spub), []byte(sysJwt2), time.Second) - require_NoError(t, err) - // test to make sure connected client (aNc) was not kicked - time.Sleep(200 * time.Millisecond) - require_True(t, len(disconnectChan) == 0) - - // ensure nothing new has happened, lookup for account not found is skipped during inc - require_True(t, atomic.LoadInt32(&reqCount) == oldRcount) - // no responders - _, err = aNc.Request("foo", nil, time.Second) - require_Error(t, err) - require_Equal(t, err.Error(), "nats: no responders available for request") - - nc2, js2 := jsClientConnect(t, sc.clusterForName("C2").randomServer(), aUsr) - defer nc2.Close() - _, err = js2.AccountInfo() - require_NoError(t, err) - - r, err = sysNc.Request(fmt.Sprintf(serverPingReqSubj, "ACCOUNTZ"), - []byte(fmt.Sprintf(`{"account":"%s"}`, spub)), time.Second) - require_NoError(t, err) - respa := ServerAPIResponse{Data: &Accountz{}} - require_NoError(t, json.Unmarshal(r.Data, &respa)) - require_True(t, hasJSExp(&respa)) - - _, err = js.AccountInfo() - require_NoError(t, err) -} diff --git a/server/leafnode_test.go b/server/leafnode_test.go index 78bd0d25..381adc2f 100644 --- a/server/leafnode_test.go +++ b/server/leafnode_test.go @@ -19,7 +19,6 @@ import ( "context" "crypto/tls" "fmt" - "io/ioutil" "math/rand" "net" "net/url" @@ -3795,239 +3794,6 @@ func TestLeafNodeNoMsgLoop(t *testing.T) { } } -func TestLeafNodeUniqueServerNameCrossJSDomain(t *testing.T) { - name := "NOT-UNIQUE" - test := func(s *Server, sIdExpected string, srvs ...*Server) { - ids := map[string]string{} - for _, srv := range srvs { - checkLeafNodeConnectedCount(t, srv, 2) - ids[srv.ID()] = srv.opts.JetStreamDomain - } - // ensure that an update for every server was received - sysNc := natsConnect(t, fmt.Sprintf("nats://admin:s3cr3t!@127.0.0.1:%d", s.opts.Port)) - defer sysNc.Close() - sub, err := sysNc.SubscribeSync(fmt.Sprintf(serverStatsSubj, "*")) - require_NoError(t, err) - for { - m, err := sub.NextMsg(time.Second) - require_NoError(t, err) - tk := strings.Split(m.Subject, ".") - if domain, ok := ids[tk[2]]; ok { - delete(ids, tk[2]) - require_Contains(t, string(m.Data), fmt.Sprintf(`"domain":"%s"`, domain)) - } - if len(ids) == 0 { - break - } - } - cnt := 0 - s.nodeToInfo.Range(func(key, value interface{}) bool { - cnt++ - require_Equal(t, value.(nodeInfo).name, name) - require_Equal(t, value.(nodeInfo).id, sIdExpected) - return true - }) - require_True(t, cnt == 1) - } - tmplA := ` - listen: -1 - server_name: %s - jetstream { - max_mem_store: 256MB, - max_file_store: 2GB, - store_dir: '%s', - domain: hub - } - accounts { - JSY { users = [ { user: "y", pass: "p" } ]; jetstream: true } - $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } - } - leaf { - port: -1 - } - ` - tmplL := ` - listen: -1 - server_name: %s - jetstream { - max_mem_store: 256MB, - max_file_store: 2GB, - store_dir: '%s', - domain: %s - } - accounts { - JSY { users = [ { user: "y", pass: "p" } ]; jetstream: true } - $SYS { users = [ { user: "admin", pass: "s3cr3t!" } ] } - } - leaf { - remotes [ - { urls: [ %s ], account: "JSY" } - { urls: [ %s ], account: "$SYS" } - ] - } - ` - t.Run("same-domain", func(t *testing.T) { - confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, name, createDir(t, JetStreamStoreDir)))) - defer removeFile(t, confA) - sA, oA := RunServerWithConfig(confA) - defer sA.Shutdown() - // using same domain as sA - confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, name, createDir(t, JetStreamStoreDir), "hub", - fmt.Sprintf("nats://y:p@127.0.0.1:%d", oA.LeafNode.Port), - fmt.Sprintf("nats://admin:s3cr3t!@127.0.0.1:%d", oA.LeafNode.Port)))) - defer removeFile(t, confL) - sL, _ := RunServerWithConfig(confL) - defer sL.Shutdown() - // as server name uniqueness is violates, sL.ID() is the expected value - test(sA, sL.ID(), sA, sL) - }) - t.Run("different-domain", func(t *testing.T) { - confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, name, createDir(t, JetStreamStoreDir)))) - defer removeFile(t, confA) - sA, oA := RunServerWithConfig(confA) - defer sA.Shutdown() - // using different domain as sA - confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, name, createDir(t, JetStreamStoreDir), "spoke", - fmt.Sprintf("nats://y:p@127.0.0.1:%d", oA.LeafNode.Port), - fmt.Sprintf("nats://admin:s3cr3t!@127.0.0.1:%d", oA.LeafNode.Port)))) - defer removeFile(t, confL) - sL, _ := RunServerWithConfig(confL) - defer sL.Shutdown() - checkLeafNodeConnectedCount(t, sL, 2) - checkLeafNodeConnectedCount(t, sA, 2) - // ensure sA contains only sA.ID - test(sA, sA.ID(), sA, sL) - }) -} - -func TestLeafNodeJwtPermsAndJetStreamDomains(t *testing.T) { - createAcc := func(js bool) (string, string, nkeys.KeyPair) { - kp, _ := nkeys.CreateAccount() - aPub, _ := kp.PublicKey() - claim := jwt.NewAccountClaims(aPub) - if js { - claim.Limits.JetStreamLimits = jwt.JetStreamLimits{ - MemoryStorage: 1024 * 1024, - DiskStorage: 1024 * 1024, - Streams: 1, Consumer: 2} - } - aJwt, err := claim.Encode(oKp) - require_NoError(t, err) - return aPub, aJwt, kp - } - sysPub, sysJwt, sysKp := createAcc(false) - accPub, accJwt, accKp := createAcc(true) - noExpiration := time.Now().Add(time.Hour) - // create user for acc to be used in leaf node. - lnCreds := createUserWithLimit(t, accKp, noExpiration, func(j *jwt.UserPermissionLimits) { - j.Sub.Deny.Add("subdeny") - j.Pub.Deny.Add("pubdeny") - }) - defer removeFile(t, lnCreds) - unlimitedCreds := createUserWithLimit(t, accKp, noExpiration, nil) - defer removeFile(t, unlimitedCreds) - - sysCreds := createUserWithLimit(t, sysKp, noExpiration, nil) - defer removeFile(t, sysCreds) - - tmplA := ` -operator: %s -system_account: %s -resolver: MEMORY -resolver_preload: { - %s: %s - %s: %s -} -listen: 127.0.0.1:-1 -leafnodes: { - listen: 127.0.0.1:-1 -} -jetstream :{ - domain: "cluster" - store_dir: '%s' - max_mem: 100Mb - max_file: 100Mb -} -` - - tmplL := ` -listen: 127.0.0.1:-1 -accounts :{ - A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, - SYS:{ users:[ {user:s1,password:s1}]}, -} -system_account = SYS -jetstream: { - domain: ln1 - store_dir: '%s' - max_mem: 50Mb - max_file: 50Mb -} -leafnodes:{ - remotes:[{ url:nats://127.0.0.1:%d, account: A, credentials: '%s'}, - { url:nats://127.0.0.1:%d, account: SYS, credentials: '%s'}] -} -` - - confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, ojwt, sysPub, - sysPub, sysJwt, accPub, accJwt, - createDir(t, JetStreamStoreDir)))) - defer removeFile(t, confA) - sA, _ := RunServerWithConfig(confA) - defer sA.Shutdown() - - confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, createDir(t, JetStreamStoreDir), - sA.opts.LeafNode.Port, lnCreds, sA.opts.LeafNode.Port, sysCreds))) - defer removeFile(t, confL) - sL, _ := RunServerWithConfig(confL) - defer sL.Shutdown() - - checkLeafNodeConnectedCount(t, sA, 2) - checkLeafNodeConnectedCount(t, sL, 2) - - ncA := natsConnect(t, sA.ClientURL(), nats.UserCredentials(unlimitedCreds)) - defer ncA.Close() - - ncL := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sL.opts.Port)) - defer ncL.Close() - - test := func(subject string, cSub, cPub *nats.Conn, remoteServerForSub *Server, accName string, pass bool) { - t.Helper() - sub, err := cSub.SubscribeSync(subject) - require_NoError(t, err) - require_NoError(t, cSub.Flush()) - // ensure the subscription made it across, or if not sent due to sub deny, make sure it could have made it. - if remoteServerForSub == nil { - time.Sleep(200 * time.Millisecond) - } else { - checkSubInterest(t, remoteServerForSub, accName, subject, time.Second) - } - require_NoError(t, cPub.Publish(subject, []byte("hello world"))) - require_NoError(t, cPub.Flush()) - m, err := sub.NextMsg(500 * time.Millisecond) - if pass { - require_NoError(t, err) - require_True(t, m.Subject == subject) - require_Equal(t, string(m.Data), "hello world") - } else { - require_True(t, err == nats.ErrTimeout) - } - } - - t.Run("sub-on-ln-pass", func(t *testing.T) { - test("sub", ncL, ncA, sA, accPub, true) - }) - t.Run("sub-on-ln-fail", func(t *testing.T) { - test("subdeny", ncL, ncA, nil, "", false) - }) - t.Run("pub-on-ln-pass", func(t *testing.T) { - test("pub", ncA, ncL, sL, "A", true) - }) - t.Run("pub-on-ln-fail", func(t *testing.T) { - test("pubdeny", ncA, ncL, nil, "A", false) - }) -} - func TestLeafNodeInterestPropagationDaisychain(t *testing.T) { aTmpl := ` port: %d @@ -4100,952 +3866,6 @@ func TestLeafNodeInterestPropagationDaisychain(t *testing.T) { checkSubInterest(t, sAA, "$G", "foo", time.Second) // failure issue 2448 } -func TestLeafNodeJetStreamClusterExtensionWithSystemAccount(t *testing.T) { - /* - Topologies tested here - same == true - A <-> B - ^ |\ - | \ - | proxy - | \ - LA <-> LB - - same == false - A <-> B - ^ ^ - | | - | proxy - | | - LA <-> LB - - The proxy is turned on later, such that the system account connection can be started later, in a controlled way - This explicitly tests the system state before and after this happens. - */ - - tmplA := ` -listen: 127.0.0.1:-1 -accounts :{ - A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, - SYS:{ users:[ {user:s1,password:s1}]}, -} -system_account: SYS -leafnodes: { - listen: 127.0.0.1:-1 - no_advertise: true - authorization: { - timeout: 0.5 - } -} -jetstream :{ - domain: "cluster" - store_dir: '%s' - max_mem: 100Mb - max_file: 100Mb -} -server_name: A -cluster: { - name: clust1 - listen: 127.0.0.1:50554 - routes=[nats-route://127.0.0.1:50555] - no_advertise: true -} -` - - tmplB := ` -listen: 127.0.0.1:-1 -accounts :{ - A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, - SYS:{ users:[ {user:s1,password:s1}]}, -} -system_account: SYS -leafnodes: { - listen: 127.0.0.1:-1 - no_advertise: true - authorization: { - timeout: 0.5 - } -} -jetstream: { - domain: "cluster" - store_dir: '%s' - max_mem: 100Mb - max_file: 100Mb -} -server_name: B -cluster: { - name: clust1 - listen: 127.0.0.1:50555 - routes=[nats-route://127.0.0.1:50554] - no_advertise: true -} -` - - tmplLA := ` -listen: 127.0.0.1:-1 -accounts :{ - A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, - SYS:{ users:[ {user:s1,password:s1}]}, -} -system_account = SYS -jetstream: { - domain: "cluster" - store_dir: '%s' - max_mem: 50Mb - max_file: 50Mb - %s -} -server_name: LA -cluster: { - name: clustL - listen: 127.0.0.1:50556 - routes=[nats-route://127.0.0.1:50557] - no_advertise: true -} -leafnodes:{ - no_advertise: true - remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A}, - {url:nats://s1:s1@127.0.0.1:%d, account: SYS}] -} -` - - tmplLB := ` -listen: 127.0.0.1:-1 -accounts :{ - A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, - SYS:{ users:[ {user:s1,password:s1}]}, -} -system_account = SYS -jetstream: { - domain: "cluster" - store_dir: '%s' - max_mem: 50Mb - max_file: 50Mb - %s -} -server_name: LB -cluster: { - name: clustL - listen: 127.0.0.1:50557 - routes=[nats-route://127.0.0.1:50556] - no_advertise: true -} -leafnodes:{ - no_advertise: true - remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A}, - {url:nats://s1:s1@127.0.0.1:%d, account: SYS}] -} -` - - for _, testCase := range []struct { - // which topology to pick - same bool - // If leaf server should be operational and form a Js cluster prior to joining. - // In this setup this would be an error as you give the wrong hint. - // But this should work itself out regardless - leafFunctionPreJoin bool - }{ - {true, true}, - {true, false}, - {false, true}, - {false, false}} { - t.Run(fmt.Sprintf("%t-%t", testCase.same, testCase.leafFunctionPreJoin), func(t *testing.T) { - sd1 := createDir(t, JetStreamStoreDir) - defer os.RemoveAll(sd1) - confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, sd1))) - defer removeFile(t, confA) - sA, _ := RunServerWithConfig(confA) - defer sA.Shutdown() - - sd2 := createDir(t, JetStreamStoreDir) - defer os.RemoveAll(sd2) - confB := createConfFile(t, []byte(fmt.Sprintf(tmplB, sd2))) - defer removeFile(t, confB) - sB, _ := RunServerWithConfig(confB) - defer sB.Shutdown() - - checkClusterFormed(t, sA, sB) - - c := cluster{t: t, servers: []*Server{sA, sB}} - c.waitOnLeader() - - // starting this will allow the second remote in tmplL to successfully connect. - port := sB.opts.LeafNode.Port - if testCase.same { - port = sA.opts.LeafNode.Port - } - p := &proxyAcceptDetectFailureLate{acceptPort: port} - defer p.close() - lPort := p.runEx(t, true) - - hint := "" - if testCase.leafFunctionPreJoin { - hint = fmt.Sprintf("extension_hint: %s", strings.ToUpper(jsNoExtend)) - } - - sd3 := createDir(t, JetStreamStoreDir) - defer os.RemoveAll(sd3) - // deliberately pick server sA and proxy - confLA := createConfFile(t, []byte(fmt.Sprintf(tmplLA, sd3, hint, sA.opts.LeafNode.Port, lPort))) - defer removeFile(t, confLA) - sLA, _ := RunServerWithConfig(confLA) - defer sLA.Shutdown() - - sd4 := createDir(t, JetStreamStoreDir) - defer os.RemoveAll(sd4) - // deliberately pick server sA and proxy - confLB := createConfFile(t, []byte(fmt.Sprintf(tmplLB, sd4, hint, sA.opts.LeafNode.Port, lPort))) - defer removeFile(t, confLB) - sLB, _ := RunServerWithConfig(confLB) - defer sLB.Shutdown() - - checkClusterFormed(t, sLA, sLB) - - strmCfg := func(name, placementCluster string) *nats.StreamConfig { - if placementCluster == "" { - return &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}} - } - return &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}, - Placement: &nats.Placement{Cluster: placementCluster}} - } - // Only after the system account is fully connected can streams be placed anywhere. - testJSFunctions := func(pass bool) { - ncA := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sA.opts.Port)) - defer ncA.Close() - jsA, err := ncA.JetStream() - require_NoError(t, err) - _, err = jsA.AddStream(strmCfg(fmt.Sprintf("fooA1-%t", pass), "")) - require_NoError(t, err) - _, err = jsA.AddStream(strmCfg(fmt.Sprintf("fooA2-%t", pass), "clust1")) - require_NoError(t, err) - _, err = jsA.AddStream(strmCfg(fmt.Sprintf("fooA3-%t", pass), "clustL")) - if pass { - require_NoError(t, err) - } else { - require_Error(t, err) - require_Contains(t, err.Error(), "insufficient resources") - } - ncL := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sLA.opts.Port)) - defer ncL.Close() - jsL, err := ncL.JetStream() - require_NoError(t, err) - _, err = jsL.AddStream(strmCfg(fmt.Sprintf("fooL1-%t", pass), "")) - require_NoError(t, err) - _, err = jsL.AddStream(strmCfg(fmt.Sprintf("fooL2-%t", pass), "clustL")) - require_NoError(t, err) - _, err = jsL.AddStream(strmCfg(fmt.Sprintf("fooL3-%t", pass), "clust1")) - if pass { - require_NoError(t, err) - } else { - require_Error(t, err) - require_Contains(t, err.Error(), "insufficient resources") - } - } - clusterLnCnt := func(expected int) error { - cnt := 0 - for _, s := range c.servers { - cnt += s.NumLeafNodes() - } - if cnt == expected { - return nil - } - return fmt.Errorf("not enought leaf node connections, got %d needed %d", cnt, expected) - } - - // Even though there are two remotes defined in tmplL, only one will be able to connect. - checkFor(t, 10*time.Second, time.Second/4, func() error { return clusterLnCnt(2) }) - checkLeafNodeConnectedCount(t, sLA, 1) - checkLeafNodeConnectedCount(t, sLB, 1) - c.waitOnPeerCount(2) - - if testCase.leafFunctionPreJoin { - cl := cluster{t: t, servers: []*Server{sLA, sLB}} - cl.waitOnLeader() - cl.waitOnPeerCount(2) - testJSFunctions(false) - } else { - // In cases where the leaf nodes have to wait for the system account to connect, - // JetStream should not be operational during that time - ncA := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sLA.opts.Port)) - defer ncA.Close() - jsA, err := ncA.JetStream() - require_NoError(t, err) - _, err = jsA.AddStream(strmCfg("fail-false", "")) - require_Error(t, err) - } - // Starting the proxy will connect the system accounts. - // After they are connected the clusters are merged. - // Once this happened, all streams in test can be placed anywhere in the cluster. - // Before that only the cluster the client is connected to can be used for placement - p.start() - - // Even though there are two remotes defined in tmplL, only one will be able to connect. - checkFor(t, 10*time.Second, time.Second/4, func() error { return clusterLnCnt(4) }) - checkLeafNodeConnectedCount(t, sLA, 2) - checkLeafNodeConnectedCount(t, sLB, 2) - - // The leader will reside in the main cluster only - c.waitOnPeerCount(4) - testJSFunctions(true) - }) - } -} - -func TestLeafNodeJetStreamClusterMixedModeExtensionWithSystemAccount(t *testing.T) { - /* Topology used in this test: - CLUSTER(A <-> B <-> C (NO JS)) - ^ - | - LA - */ - - // once every server is up, we expect these peers to be part of the JetStream meta cluster - expectedJetStreamPeers := map[string]struct{}{ - "A": {}, - "B": {}, - "LA": {}, - } - - tmplA := ` -listen: 127.0.0.1:-1 -accounts :{ - A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, - SYS:{ users:[ {user:s1,password:s1}]}, -} -system_account: SYS -leafnodes: { - listen: 127.0.0.1:-1 - no_advertise: true - authorization: { - timeout: 0.5 - } -} -jetstream: { %s store_dir: '%s'; max_mem: 50Mb, max_file: 50Mb } -server_name: A -cluster: { - name: clust1 - listen: 127.0.0.1:50554 - routes=[nats-route://127.0.0.1:50555,nats-route://127.0.0.1:50556] - no_advertise: true -} -` - - tmplB := ` -listen: 127.0.0.1:-1 -accounts :{ - A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, - SYS:{ users:[ {user:s1,password:s1}]}, -} -system_account: SYS -leafnodes: { - listen: 127.0.0.1:-1 - no_advertise: true - authorization: { - timeout: 0.5 - } -} -jetstream: { %s store_dir: '%s'; max_mem: 50Mb, max_file: 50Mb } -server_name: B -cluster: { - name: clust1 - listen: 127.0.0.1:50555 - routes=[nats-route://127.0.0.1:50554,nats-route://127.0.0.1:50556] - no_advertise: true -} -` - - tmplC := ` -listen: 127.0.0.1:-1 -accounts :{ - A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, - SYS:{ users:[ {user:s1,password:s1}]}, -} -system_account: SYS -leafnodes: { - listen: 127.0.0.1:-1 - no_advertise: true - authorization: { - timeout: 0.5 - } -} -jetstream: { - enabled: false - %s -} -server_name: C -cluster: { - name: clust1 - listen: 127.0.0.1:50556 - routes=[nats-route://127.0.0.1:50554,nats-route://127.0.0.1:50555] - no_advertise: true -} -` - - tmplLA := ` -listen: 127.0.0.1:-1 -accounts :{ - A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, - SYS:{ users:[ {user:s1,password:s1}]}, -} -system_account = SYS -# the extension hint is to simplify this test. without it present we would need a cluster of size 2 -jetstream: { %s store_dir: '%s'; max_mem: 50Mb, max_file: 50Mb, extension_hint: will_extend } -server_name: LA -leafnodes:{ - no_advertise: true - remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A}, - {url:nats://s1:s1@127.0.0.1:%d, account: SYS}] -} -# add the cluster here so we can test placement -cluster: { name: clustL } -` - for _, withDomain := range []bool{true, false} { - t.Run(fmt.Sprintf("with-domain:%t", withDomain), func(t *testing.T) { - jsDisabledDomainString := _EMPTY_ - jsEnabledDomainString := _EMPTY_ - if withDomain { - jsEnabledDomainString = `domain: "domain", ` - jsDisabledDomainString = `domain: "domain"` - } else { - // in case no domain name is set, fall back to the extension hint. - // since JS is disabled, the value of this does not clash with other uses. - jsDisabledDomainString = "extension_hint: will_extend" - } - - sd1 := createDir(t, JetStreamStoreDir) - defer os.RemoveAll(sd1) - confA := createConfFile(t, []byte(fmt.Sprintf(tmplA, jsEnabledDomainString, sd1))) - defer removeFile(t, confA) - sA, _ := RunServerWithConfig(confA) - defer sA.Shutdown() - - sd2 := createDir(t, JetStreamStoreDir) - defer os.RemoveAll(sd2) - confB := createConfFile(t, []byte(fmt.Sprintf(tmplB, jsEnabledDomainString, sd2))) - defer removeFile(t, confB) - sB, _ := RunServerWithConfig(confB) - defer sB.Shutdown() - - confC := createConfFile(t, []byte(fmt.Sprintf(tmplC, jsDisabledDomainString))) - defer removeFile(t, confC) - sC, _ := RunServerWithConfig(confC) - defer sC.Shutdown() - - checkClusterFormed(t, sA, sB, sC) - c := cluster{t: t, servers: []*Server{sA, sB, sC}} - c.waitOnPeerCount(2) - - sd3 := createDir(t, JetStreamStoreDir) - defer os.RemoveAll(sd3) - // deliberately pick server sC (no JS) to connect to - confLA := createConfFile(t, []byte(fmt.Sprintf(tmplLA, jsEnabledDomainString, sd3, sC.opts.LeafNode.Port, sC.opts.LeafNode.Port))) - defer removeFile(t, confLA) - sLA, _ := RunServerWithConfig(confLA) - defer sLA.Shutdown() - - checkLeafNodeConnectedCount(t, sC, 2) - checkLeafNodeConnectedCount(t, sLA, 2) - c.waitOnPeerCount(3) - peers := c.leader().JetStreamClusterPeers() - for _, peer := range peers { - if _, ok := expectedJetStreamPeers[peer]; !ok { - t.Fatalf("Found unexpected peer %q", peer) - } - } - - // helper to create stream config with uniqe name and subject - cnt := 0 - strmCfg := func(placementCluster string) *nats.StreamConfig { - name := fmt.Sprintf("s-%d", cnt) - cnt++ - if placementCluster == "" { - return &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}} - } - return &nats.StreamConfig{Name: name, Replicas: 1, Subjects: []string{name}, - Placement: &nats.Placement{Cluster: placementCluster}} - } - - test := func(port int, expectedDefPlacement string) { - ncA := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", port)) - defer ncA.Close() - jsA, err := ncA.JetStream() - require_NoError(t, err) - si, err := jsA.AddStream(strmCfg("")) - require_NoError(t, err) - require_Contains(t, si.Cluster.Name, expectedDefPlacement) - si, err = jsA.AddStream(strmCfg("clust1")) - require_NoError(t, err) - require_Contains(t, si.Cluster.Name, "clust1") - si, err = jsA.AddStream(strmCfg("clustL")) - require_NoError(t, err) - require_Contains(t, si.Cluster.Name, "clustL") - } - - test(sA.opts.Port, "clust1") - test(sB.opts.Port, "clust1") - test(sC.opts.Port, "clust1") - test(sLA.opts.Port, "clustL") - }) - } -} - -func TestLeafNodeJetStreamCredsDenies(t *testing.T) { - tmplL := ` -listen: 127.0.0.1:-1 -accounts :{ - A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, - SYS:{ users:[ {user:s1,password:s1}]}, -} -system_account = SYS -jetstream: { - domain: "cluster" - store_dir: '%s' - max_mem: 50Mb - max_file: 50Mb -} -leafnodes:{ - remotes:[{url:nats://a1:a1@127.0.0.1:50555, account: A, credentials: '%s' }, - {url:nats://s1:s1@127.0.0.1:50555, account: SYS, credentials: '%s', deny_imports: foo, deny_exports: bar}] -} -` - akp, err := nkeys.CreateAccount() - require_NoError(t, err) - creds := createUserWithLimit(t, akp, time.Time{}, func(pl *jwt.UserPermissionLimits) { - pl.Pub.Deny.Add(jsAllAPI) - pl.Sub.Deny.Add(jsAllAPI) - }) - - sd := createDir(t, JetStreamStoreDir) - defer os.RemoveAll(sd) - - confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, sd, creds, creds))) - defer removeFile(t, confL) - opts := LoadConfig(confL) - sL, err := NewServer(opts) - require_NoError(t, err) - - l := captureNoticeLogger{} - sL.SetLogger(&l, false, false) - - go sL.Start() - defer sL.Shutdown() - - // wait till the notices got printed -UNTIL_READY: - for { - <-time.After(50 * time.Millisecond) - l.Lock() - for _, n := range l.notices { - if strings.Contains(n, "Server is ready") { - l.Unlock() - break UNTIL_READY - } - } - l.Unlock() - } - - l.Lock() - cnt := 0 - for _, n := range l.notices { - if strings.Contains(n, "LeafNode Remote for Account A uses credentials file") || - strings.Contains(n, "LeafNode Remote for System Account uses") || - strings.Contains(n, "Remote for System Account uses restricted export permissions") || - strings.Contains(n, "Remote for System Account uses restricted import permissions") { - cnt++ - } - } - l.Unlock() - require_True(t, cnt == 4) -} - -func TestLeafNodeJetStreamDefaultDomainCfg(t *testing.T) { - tmplHub := ` -listen: 127.0.0.1:%d -accounts :{ - A:{ jetstream: %s, users:[ {user:a1,password:a1}]}, - SYS:{ users:[ {user:s1,password:s1}]}, -} -system_account: SYS -jetstream : %s -server_name: HUB -leafnodes: { - listen: 127.0.0.1:%d -} -%s -` - - tmplL := ` -listen: 127.0.0.1:-1 -accounts :{ - A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, - SYS:{ users:[ {user:s1,password:s1}]}, -} -system_account: SYS -jetstream: { domain: "%s", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } -server_name: LEAF -leafnodes: { - remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},%s] -} -%s -` - - test := func(domain string, sysShared bool) { - confHub := createConfFile(t, []byte(fmt.Sprintf(tmplHub, -1, "disabled", "disabled", -1, ""))) - defer removeFile(t, confHub) - sHub, _ := RunServerWithConfig(confHub) - defer sHub.Shutdown() - - noDomainFix := "" - if domain == _EMPTY_ { - noDomainFix = `default_js_domain:{A:""}` - } - - sys := "" - if sysShared { - sys = fmt.Sprintf(`{url:nats://s1:s1@127.0.0.1:%d, account: SYS}`, sHub.opts.LeafNode.Port) - } - - sdLeaf := createDir(t, JetStreamStoreDir) - defer os.RemoveAll(sdLeaf) - confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, domain, sdLeaf, sHub.opts.LeafNode.Port, sys, noDomainFix))) - defer removeFile(t, confL) - sLeaf, _ := RunServerWithConfig(confL) - defer sLeaf.Shutdown() - - lnCnt := 1 - if sysShared { - lnCnt++ - } - - checkLeafNodeConnectedCount(t, sHub, lnCnt) - checkLeafNodeConnectedCount(t, sLeaf, lnCnt) - - ncA := natsConnect(t, fmt.Sprintf("nats://a1:a1@127.0.0.1:%d", sHub.opts.Port)) - defer ncA.Close() - jsA, err := ncA.JetStream() - require_NoError(t, err) - - _, err = jsA.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, Subjects: []string{"foo"}}) - require_True(t, err == nats.ErrNoResponders) - - // Add in default domain and restart server - require_NoError(t, ioutil.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub, - sHub.opts.Port, - "disabled", - "disabled", - sHub.opts.LeafNode.Port, - fmt.Sprintf(`default_js_domain: {A:"%s"}`, domain))), 0664)) - - sHub.Shutdown() - sHub.WaitForShutdown() - checkLeafNodeConnectedCount(t, sLeaf, 0) - sHubUpd1, _ := RunServerWithConfig(confHub) - defer sHubUpd1.Shutdown() - - checkLeafNodeConnectedCount(t, sHubUpd1, lnCnt) - checkLeafNodeConnectedCount(t, sLeaf, lnCnt) - - _, err = jsA.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, Subjects: []string{"foo"}}) - require_NoError(t, err) - - // Enable jetstream in hub. - sdHub := createDir(t, JetStreamStoreDir) - defer os.RemoveAll(sdHub) - jsEnabled := fmt.Sprintf(`{ domain: "%s", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb }`, domain, sdHub) - require_NoError(t, ioutil.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub, - sHubUpd1.opts.Port, - "disabled", - jsEnabled, - sHubUpd1.opts.LeafNode.Port, - fmt.Sprintf(`default_js_domain: {A:"%s"}`, domain))), 0664)) - - sHubUpd1.Shutdown() - sHubUpd1.WaitForShutdown() - checkLeafNodeConnectedCount(t, sLeaf, 0) - sHubUpd2, _ := RunServerWithConfig(confHub) - defer sHubUpd2.Shutdown() - - checkLeafNodeConnectedCount(t, sHubUpd2, lnCnt) - checkLeafNodeConnectedCount(t, sLeaf, lnCnt) - - _, err = jsA.AddStream(&nats.StreamConfig{Name: "bar", Replicas: 1, Subjects: []string{"bar"}}) - require_NoError(t, err) - - // Enable jetstream in account A of hub - // This is a mis config, as you can't have it both ways, local jetstream but default to another one - require_NoError(t, ioutil.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub, - sHubUpd2.opts.Port, - "enabled", - jsEnabled, - sHubUpd2.opts.LeafNode.Port, - fmt.Sprintf(`default_js_domain: {A:"%s"}`, domain))), 0664)) - - if domain != _EMPTY_ { - // in case no domain name exists there are no additional guard rails, hence no error - // It is the users responsibility to get this edge case right - sHubUpd2.Shutdown() - sHubUpd2.WaitForShutdown() - checkLeafNodeConnectedCount(t, sLeaf, 0) - sHubUpd3, err := NewServer(LoadConfig(confHub)) - sHubUpd3.Shutdown() - - require_Error(t, err) - require_Contains(t, err.Error(), `default_js_domain contains account name "A" with enabled JetStream`) - } - } - - t.Run("with-domain-sys", func(t *testing.T) { - test("domain", true) - }) - t.Run("with-domain-nosys", func(t *testing.T) { - test("domain", false) - }) - t.Run("no-domain", func(t *testing.T) { - test("", true) - }) - t.Run("no-domain", func(t *testing.T) { - test("", false) - }) -} - -func TestLeafNodeJetStreamDefaultDomainJwtExplicit(t *testing.T) { - tmplHub := ` -listen: 127.0.0.1:%d -operator: %s -system_account: %s -resolver: MEM -resolver_preload: { - %s:%s - %s:%s -} -jetstream : disabled -server_name: HUB -leafnodes: { - listen: 127.0.0.1:%d -} -%s -` - - tmplL := ` -listen: 127.0.0.1:-1 -accounts :{ - A:{ jetstream: enable, users:[ {user:a1,password:a1}]}, - SYS:{ users:[ {user:s1,password:s1}]}, -} -system_account: SYS -jetstream: { domain: "%s", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } -server_name: LEAF -leafnodes: { - remotes:[{url:nats://127.0.0.1:%d, account: A, credentials: '%s'}, - {url:nats://127.0.0.1:%d, account: SYS, credentials: '%s'}] -} -%s -` - - test := func(domain string) { - noDomainFix := "" - if domain == _EMPTY_ { - noDomainFix = `default_js_domain:{A:""}` - } - - sysKp, syspub := createKey(t) - sysJwt := encodeClaim(t, jwt.NewAccountClaims(syspub), syspub) - sysCreds := newUser(t, sysKp) - defer removeFile(t, sysCreds) - - aKp, aPub := createKey(t) - aClaim := jwt.NewAccountClaims(aPub) - aJwt := encodeClaim(t, aClaim, aPub) - aCreds := newUser(t, aKp) - defer removeFile(t, aCreds) - - confHub := createConfFile(t, []byte(fmt.Sprintf(tmplHub, -1, ojwt, syspub, syspub, sysJwt, aPub, aJwt, -1, ""))) - defer removeFile(t, confHub) - sHub, _ := RunServerWithConfig(confHub) - defer sHub.Shutdown() - - sdLeaf := createDir(t, JetStreamStoreDir) - defer os.RemoveAll(sdLeaf) - confL := createConfFile(t, []byte(fmt.Sprintf(tmplL, - domain, - sdLeaf, - sHub.opts.LeafNode.Port, - aCreds, - sHub.opts.LeafNode.Port, - sysCreds, - noDomainFix))) - defer removeFile(t, confL) - sLeaf, _ := RunServerWithConfig(confL) - defer sLeaf.Shutdown() - - checkLeafNodeConnectedCount(t, sHub, 2) - checkLeafNodeConnectedCount(t, sLeaf, 2) - - ncA := natsConnect(t, fmt.Sprintf("nats://127.0.0.1:%d", sHub.opts.Port), createUserCreds(t, nil, aKp)) - defer ncA.Close() - jsA, err := ncA.JetStream() - require_NoError(t, err) - - _, err = jsA.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, Subjects: []string{"foo"}}) - require_True(t, err == nats.ErrNoResponders) - - // Add in default domain and restart server - require_NoError(t, ioutil.WriteFile(confHub, []byte(fmt.Sprintf(tmplHub, - sHub.opts.Port, ojwt, syspub, syspub, sysJwt, aPub, aJwt, sHub.opts.LeafNode.Port, - fmt.Sprintf(`default_js_domain: {%s:"%s"}`, aPub, domain))), 0664)) - - sHub.Shutdown() - sHub.WaitForShutdown() - checkLeafNodeConnectedCount(t, sLeaf, 0) - sHubUpd1, _ := RunServerWithConfig(confHub) - defer sHubUpd1.Shutdown() - - checkLeafNodeConnectedCount(t, sHubUpd1, 2) - checkLeafNodeConnectedCount(t, sLeaf, 2) - - _, err = jsA.AddStream(&nats.StreamConfig{Name: "bar", Replicas: 1, Subjects: []string{"bar"}}) - require_NoError(t, err) - } - t.Run("with-domain", func(t *testing.T) { - test("domain") - }) - t.Run("no-domain", func(t *testing.T) { - test("") - }) -} - -func TestLeafNodeJetStreamDefaultDomainClusterBothEnds(t *testing.T) { - // test to ensure that default domain functions when both ends of the leaf node connection are clusters - tmplHub1 := ` -listen: 127.0.0.1:-1 -accounts :{ - A:{ jetstream: enabled, users:[ {user:a1,password:a1}]}, - B:{ jetstream: enabled, users:[ {user:b1,password:b1}]} -} -jetstream : { domain: "DHUB", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } -server_name: HUB1 -cluster: { - name: HUB - listen: 127.0.0.1:50554 - routes=[nats-route://127.0.0.1:50555] -} -leafnodes: { - listen:127.0.0.1:-1 -} -` - - tmplHub2 := ` -listen: 127.0.0.1:-1 -accounts :{ - A:{ jetstream: enabled, users:[ {user:a1,password:a1}]}, - B:{ jetstream: enabled, users:[ {user:b1,password:b1}]} -} -jetstream : { domain: "DHUB", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } -server_name: HUB2 -cluster: { - name: HUB - listen: 127.0.0.1:50555 - routes=[nats-route://127.0.0.1:50554] -} -leafnodes: { - listen:127.0.0.1:-1 -} -` - - tmplL1 := ` -listen: 127.0.0.1:-1 -accounts :{ - A:{ jetstream: enabled, users:[ {user:a1,password:a1}]}, - B:{ jetstream: disabled, users:[ {user:b1,password:b1}]} -} -jetstream: { domain: "DLEAF", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } -server_name: LEAF1 -cluster: { - name: LEAF - listen: 127.0.0.1:50556 - routes=[nats-route://127.0.0.1:50557] -} -leafnodes: { - remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},{url:nats://b1:b1@127.0.0.1:%d, account: B}] -} -default_js_domain: {B:"DHUB"} -` - - tmplL2 := ` -listen: 127.0.0.1:-1 -accounts :{ - A:{ jetstream: enabled, users:[ {user:a1,password:a1}]}, - B:{ jetstream: disabled, users:[ {user:b1,password:b1}]} -} -jetstream: { domain: "DLEAF", store_dir: '%s', max_mem: 100Mb, max_file: 100Mb } -server_name: LEAF2 -cluster: { - name: LEAF - listen: 127.0.0.1:50557 - routes=[nats-route://127.0.0.1:50556] -} -leafnodes: { - remotes:[{url:nats://a1:a1@127.0.0.1:%d, account: A},{url:nats://b1:b1@127.0.0.1:%d, account: B}] -} -default_js_domain: {B:"DHUB"} -` - - sd1 := createDir(t, JetStreamStoreDir) - defer os.RemoveAll(sd1) - confHub1 := createConfFile(t, []byte(fmt.Sprintf(tmplHub1, sd1))) - defer removeFile(t, confHub1) - sHub1, _ := RunServerWithConfig(confHub1) - defer sHub1.Shutdown() - - sd2 := createDir(t, JetStreamStoreDir) - defer os.RemoveAll(sd2) - confHub2 := createConfFile(t, []byte(fmt.Sprintf(tmplHub2, sd2))) - defer removeFile(t, confHub2) - sHub2, _ := RunServerWithConfig(confHub2) - defer sHub2.Shutdown() - - checkClusterFormed(t, sHub1, sHub2) - c1 := cluster{t: t, servers: []*Server{sHub1, sHub2}} - c1.waitOnPeerCount(2) - - sd3 := createDir(t, JetStreamStoreDir) - defer os.RemoveAll(sd3) - confLeaf1 := createConfFile(t, []byte(fmt.Sprintf(tmplL1, sd3, sHub1.getOpts().LeafNode.Port, sHub1.getOpts().LeafNode.Port))) - defer removeFile(t, confLeaf1) - sLeaf1, _ := RunServerWithConfig(confLeaf1) - defer sLeaf1.Shutdown() - - sd4 := createDir(t, JetStreamStoreDir) - defer os.RemoveAll(sd4) - confLeaf2 := createConfFile(t, []byte(fmt.Sprintf(tmplL2, sd3, sHub1.getOpts().LeafNode.Port, sHub1.getOpts().LeafNode.Port))) - defer removeFile(t, confLeaf2) - sLeaf2, _ := RunServerWithConfig(confLeaf2) - defer sLeaf2.Shutdown() - - checkClusterFormed(t, sLeaf1, sLeaf2) - c2 := cluster{t: t, servers: []*Server{sLeaf1, sLeaf2}} - c2.waitOnPeerCount(2) - - checkLeafNodeConnectedCount(t, sHub1, 4) - checkLeafNodeConnectedCount(t, sLeaf1, 2) - checkLeafNodeConnectedCount(t, sLeaf2, 2) - - ncB := natsConnect(t, fmt.Sprintf("nats://b1:b1@127.0.0.1:%d", sLeaf1.getOpts().Port)) - defer ncB.Close() - jsB1, err := ncB.JetStream() - require_NoError(t, err) - si, err := jsB1.AddStream(&nats.StreamConfig{Name: "foo", Replicas: 1, Subjects: []string{"foo"}}) - require_NoError(t, err) - require_Equal(t, si.Cluster.Name, "HUB") - - jsB2, err := ncB.JetStream(nats.Domain("DHUB")) - require_NoError(t, err) - si, err = jsB2.AddStream(&nats.StreamConfig{Name: "bar", Replicas: 1, Subjects: []string{"bar"}}) - require_NoError(t, err) - require_Equal(t, si.Cluster.Name, "HUB") - -} - func TestLeafNodeQueueGroupWithLateLNJoin(t *testing.T) { /* diff --git a/server/norace_test.go b/server/norace_test.go index f10ae039..564e6345 100644 --- a/server/norace_test.go +++ b/server/norace_test.go @@ -11,8 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !race -// +build !race +//go:build !race && !skip_no_race_tests +// +build !race,!skip_no_race_tests package server diff --git a/test/fanout_test.go b/test/fanout_test.go index 54c98afb..ab7a39cd 100644 --- a/test/fanout_test.go +++ b/test/fanout_test.go @@ -11,8 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !race -// +build !race +//go:build !race && !skipnoracetests +// +build !race,!skipnoracetests package test diff --git a/test/norace_test.go b/test/norace_test.go index 871f170d..b26dc266 100644 --- a/test/norace_test.go +++ b/test/norace_test.go @@ -11,8 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !race -// +build !race +//go:build !race && !skip_no_race_tests +// +build !race,!skip_no_race_tests package test