From a6f7f5476ecee06a4ad87ec52e9cee2b950396ba Mon Sep 17 00:00:00 2001 From: Sylvain Rabot Date: Mon, 10 Jul 2023 21:56:53 +0200 Subject: [PATCH 1/5] Bump v2.9 CI and nightlies to Go 1.20 Signed-off-by: Waldemar Quevedo --- .github/actions/nightly-release/action.yaml | 2 +- .github/workflows/cov.yaml | 4 ++-- .github/workflows/go-test.yaml | 15 ++++----------- .github/workflows/nightly.yaml | 2 +- .github/workflows/rc_nightly.yaml | 2 +- .travis.yml | 4 +++- docker/Dockerfile.nightly | 2 +- scripts/runTestsOnTravis.sh | 6 +----- 8 files changed, 14 insertions(+), 23 deletions(-) diff --git a/.github/actions/nightly-release/action.yaml b/.github/actions/nightly-release/action.yaml index 7fab7dd6..2849702b 100644 --- a/.github/actions/nightly-release/action.yaml +++ b/.github/actions/nightly-release/action.yaml @@ -26,7 +26,7 @@ runs: using: composite steps: - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: "${{ inputs.go }}" diff --git a/.github/workflows/cov.yaml b/.github/workflows/cov.yaml index d878f62f..39134dd2 100644 --- a/.github/workflows/cov.yaml +++ b/.github/workflows/cov.yaml @@ -21,9 +21,9 @@ jobs: ref: dev - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: - go-version: 1.19.x + go-version: "1.20.x" - name: Run code coverage shell: bash --noprofile --norc -x -eo pipefail {0} diff --git a/.github/workflows/go-test.yaml b/.github/workflows/go-test.yaml index bac6c719..d3dbae9d 100644 --- a/.github/workflows/go-test.yaml +++ b/.github/workflows/go-test.yaml @@ -5,8 +5,7 @@ jobs: test: strategy: matrix: - go: [1.16] - + go: ["1.20"] env: GOPATH: /home/runner/work/nats-server GO111MODULE: "on" @@ -19,25 +18,19 @@ jobs: path: src/github.com/nats-io/nats-server - name: Setup Go - uses: actions/setup-go@v1 + uses: actions/setup-go@v4 with: go-version: ${{matrix.go}} - name: Install deps shell: bash --noprofile --norc -x -eo pipefail {0} run: | - go get -u honnef.co/go/tools/cmd/staticcheck - go get -u github.com/client9/misspell/cmd/misspell + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.53.3 - name: Lint shell: bash --noprofile --norc -x -eo pipefail {0} run: | - GO_LIST=$(go list ./...) - go build - $(exit $(go fmt $GO_LIST | wc -l)) - go vet $GO_LIST - find . -type f -name "*.go" | xargs $GOPATH/bin/misspell -error -locale US - $GOPATH/bin/staticcheck $GO_LIST + golangci-lint run - name: Run tests shell: bash --noprofile --norc -x -eo pipefail {0} diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 61433fe2..8f2462eb 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -22,7 +22,7 @@ jobs: - uses: ./src/github.com/nats-io/nats-server/.github/actions/nightly-release with: - go: "1.19" + go: "1.20" workdir: src/github.com/nats-io/nats-server label: nightly hub_username: "${{ secrets.DOCKER_USERNAME }}" diff --git a/.github/workflows/rc_nightly.yaml b/.github/workflows/rc_nightly.yaml index 2228e226..f6cf6bc7 100644 --- a/.github/workflows/rc_nightly.yaml +++ b/.github/workflows/rc_nightly.yaml @@ -22,7 +22,7 @@ jobs: - uses: ./src/github.com/nats-io/nats-server/.github/actions/nightly-release with: - go: "1.19" + go: "1.20" workdir: src/github.com/nats-io/nats-server label: nightly-main hub_username: "${{ secrets.DOCKER_USERNAME }}" diff --git a/.travis.yml b/.travis.yml index 75e7d2b1..2592c26f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,9 @@ vm: language: go go: + - "1.20.x" - "1.19.12" + go_import_path: github.com/nats-io/nats-server addons: @@ -48,4 +50,4 @@ deploy: script: curl -sL http://git.io/goreleaser | bash on: tags: true - condition: ($TRAVIS_GO_VERSION =~ 1.19) && ($TEST_SUITE = "compile") + condition: ($TRAVIS_GO_VERSION =~ 1.20) && ($TEST_SUITE = "compile") diff --git a/docker/Dockerfile.nightly b/docker/Dockerfile.nightly index 87f6d950..aa2ade34 100644 --- a/docker/Dockerfile.nightly +++ b/docker/Dockerfile.nightly @@ -1,4 +1,4 @@ -FROM golang:1.19-alpine AS builder +FROM golang:1.20-alpine AS builder ARG VERSION="nightly" diff --git a/scripts/runTestsOnTravis.sh b/scripts/runTestsOnTravis.sh index eb21304f..53961a5b 100755 --- a/scripts/runTestsOnTravis.sh +++ b/scripts/runTestsOnTravis.sh @@ -7,11 +7,7 @@ if [ "$1" = "compile" ]; then go build; # Now run the linters. - # TODO: Pinning a specific commit here as there is a bugfix merged that - # fixes gofmt on macOS Ventura, we can undo this and go back to the binary - # install script once there's a new tagged release that contains the fix. - # curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.50.1 - go install github.com/golangci/golangci-lint/cmd/golangci-lint@6f7f8ae; + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.53.3; golangci-lint run; if [ "$TRAVIS_TAG" != "" ]; then go test -race -v -run=TestVersionMatchesTag ./server -count=1 -vet=off From d8bdbc34c44d440936b57542901d87d1020ceb2d Mon Sep 17 00:00:00 2001 From: Waldemar Quevedo Date: Mon, 28 Aug 2023 10:09:46 -0700 Subject: [PATCH 2/5] Fix resetting TLS name from solicited remotes In +Go 1.20, the x509.HostnameError changed to be wrapped in a tls.CertificateVerificationError so sometimes the name would not be reset. This would make the `TestGatewayTLSMixedIPAndDNS` test flake sometimes. Signed-off-by: Waldemar Quevedo --- server/client.go | 6 +- test/gateway_test.go | 159 ++++++++++++++++++++++--------------------- 2 files changed, 88 insertions(+), 77 deletions(-) diff --git a/server/client.go b/server/client.go index b6f73442..e98c6054 100644 --- a/server/client.go +++ b/server/client.go @@ -18,6 +18,7 @@ import ( "crypto/tls" "crypto/x509" "encoding/json" + "errors" "fmt" "io" "math/rand" @@ -5304,7 +5305,10 @@ func (c *client) doTLSHandshake(typ string, solicit bool, url *url.URL, tlsConfi if solicit { // Based on type of error, possibly clear the saved tlsName // See: https://github.com/nats-io/nats-server/issues/1256 - if _, ok := err.(x509.HostnameError); ok { + // NOTE: As of Go 1.20, the HostnameError is wrapped so cannot + // type assert to check directly. + var hostnameErr x509.HostnameError + if errors.As(err, &hostnameErr) { if host == tlsName { resetTLSName = true } diff --git a/test/gateway_test.go b/test/gateway_test.go index 3975a148..49e7274c 100644 --- a/test/gateway_test.go +++ b/test/gateway_test.go @@ -605,89 +605,96 @@ func TestGatewayTLSMixedIPAndDNS(t *testing.T) { server.SetGatewaysSolicitDelay(5 * time.Millisecond) defer server.ResetGatewaysSolicitDelay() - confA1 := createConfFile(t, []byte(` - listen: 127.0.0.1:-1 - gateway { - name: "A" - listen: "127.0.0.1:-1" - tls { - cert_file: "./configs/certs/server-iponly.pem" - key_file: "./configs/certs/server-key-iponly.pem" - ca_file: "./configs/certs/ca.pem" - timeout: 2 + // Run this test extra times to make sure not flaky since it + // on solicit time. + for i := 0; i < 10; i++ { + t.Run("", func(t *testing.T) { + confA1 := createConfFile(t, []byte(` + listen: 127.0.0.1:-1 + server_name: A1 + gateway { + name: "A" + listen: "127.0.0.1:-1" + tls { + cert_file: "./configs/certs/server-iponly.pem" + key_file: "./configs/certs/server-key-iponly.pem" + ca_file: "./configs/certs/ca.pem" + timeout: 2 + } } - } - cluster { - listen: "127.0.0.1:-1" - } - `)) - srvA1, optsA1 := RunServerWithConfig(confA1) - defer srvA1.Shutdown() + cluster { + listen: "127.0.0.1:-1" + }`)) + srvA1, optsA1 := RunServerWithConfig(confA1) + defer srvA1.Shutdown() - confA2Template := ` - listen: 127.0.0.1:-1 - gateway { - name: "A" - listen: "localhost:-1" - tls { - cert_file: "./configs/certs/server-cert.pem" - key_file: "./configs/certs/server-key.pem" - ca_file: "./configs/certs/ca.pem" - timeout: 2 + confA2Template := ` + listen: 127.0.0.1:-1 + server_name: A2 + gateway { + name: "A" + listen: "localhost:-1" + tls { + cert_file: "./configs/certs/server-cert.pem" + key_file: "./configs/certs/server-key.pem" + ca_file: "./configs/certs/ca.pem" + timeout: 2 + } } - } - cluster { - listen: "127.0.0.1:-1" - routes [ - "nats://%s:%d" - ] - } - ` - confA2 := createConfFile(t, []byte(fmt.Sprintf(confA2Template, - optsA1.Cluster.Host, optsA1.Cluster.Port))) - srvA2, optsA2 := RunServerWithConfig(confA2) - defer srvA2.Shutdown() + cluster { + listen: "127.0.0.1:-1" + routes [ + "nats://%s:%d" + ] + }` + confA2 := createConfFile(t, []byte(fmt.Sprintf(confA2Template, + optsA1.Cluster.Host, optsA1.Cluster.Port))) + srvA2, optsA2 := RunServerWithConfig(confA2) + defer srvA2.Shutdown() - checkClusterFormed(t, srvA1, srvA2) + checkClusterFormed(t, srvA1, srvA2) - // Create a GW connection to cluster "A". Don't use the helper since we need verification etc. - o := DefaultTestOptions - o.Port = -1 - o.Gateway.Name = "B" - o.Gateway.Host = "127.0.0.1" - o.Gateway.Port = -1 + // Create a GW connection to cluster "A". Don't use the helper since we need verification etc. + o := DefaultTestOptions + o.Port = -1 + o.ServerName = "B1" + o.Gateway.Name = "B" + o.Gateway.Host = "127.0.0.1" + o.Gateway.Port = -1 - tc := &server.TLSConfigOpts{} - tc.CertFile = "./configs/certs/server-cert.pem" - tc.KeyFile = "./configs/certs/server-key.pem" - tc.CaFile = "./configs/certs/ca.pem" - tc.Timeout = 2.0 - tlsConfig, err := server.GenTLSConfig(tc) - if err != nil { - t.Fatalf("Error generating TLS config: %v", err) + tc := &server.TLSConfigOpts{} + tc.CertFile = "./configs/certs/server-cert.pem" + tc.KeyFile = "./configs/certs/server-key.pem" + tc.CaFile = "./configs/certs/ca.pem" + tc.Timeout = 2.0 + tlsConfig, err := server.GenTLSConfig(tc) + if err != nil { + t.Fatalf("Error generating TLS config: %v", err) + } + tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert + tlsConfig.RootCAs = tlsConfig.ClientCAs + + o.Gateway.TLSConfig = tlsConfig.Clone() + + rurl, _ := url.Parse(fmt.Sprintf("nats://%s:%d", optsA2.Gateway.Host, optsA2.Gateway.Port)) + remote := &server.RemoteGatewayOpts{Name: "A", URLs: []*url.URL{rurl}} + remote.TLSConfig = tlsConfig.Clone() + o.Gateway.Gateways = []*server.RemoteGatewayOpts{remote} + + srvB := RunServer(&o) + defer srvB.Shutdown() + + waitForOutboundGateways(t, srvB, 1, 10*time.Second) + waitForOutboundGateways(t, srvA1, 1, 10*time.Second) + waitForOutboundGateways(t, srvA2, 1, 10*time.Second) + + // Now kill off srvA2 and force serverB to connect to srvA1. + srvA2.Shutdown() + + // Make sure this works. + waitForOutboundGateways(t, srvB, 1, 30*time.Second) + }) } - tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert - tlsConfig.RootCAs = tlsConfig.ClientCAs - - o.Gateway.TLSConfig = tlsConfig.Clone() - - rurl, _ := url.Parse(fmt.Sprintf("nats://%s:%d", optsA2.Gateway.Host, optsA2.Gateway.Port)) - remote := &server.RemoteGatewayOpts{Name: "A", URLs: []*url.URL{rurl}} - remote.TLSConfig = tlsConfig.Clone() - o.Gateway.Gateways = []*server.RemoteGatewayOpts{remote} - - srvB := RunServer(&o) - defer srvB.Shutdown() - - waitForOutboundGateways(t, srvB, 1, 10*time.Second) - waitForOutboundGateways(t, srvA1, 1, 10*time.Second) - waitForOutboundGateways(t, srvA2, 1, 10*time.Second) - - // Now kill off srvA2 and force serverB to connect to srvA1. - srvA2.Shutdown() - - // Make sure this works. - waitForOutboundGateways(t, srvB, 1, 10*time.Second) } func TestGatewayAdvertiseInCluster(t *testing.T) { From 7a2020a34755829f3f02bffcfccde82bc86c9286 Mon Sep 17 00:00:00 2001 From: Waldemar Quevedo Date: Mon, 4 Sep 2023 12:55:06 -0700 Subject: [PATCH 3/5] Remove Go 1.19 build Signed-off-by: Waldemar Quevedo --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2592c26f..6607183d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ vm: language: go go: - "1.20.x" - - "1.19.12" go_import_path: github.com/nats-io/nats-server From d29df72f83dcc7ebcd89f038d3373630313d0660 Mon Sep 17 00:00:00 2001 From: Byron Ruth Date: Wed, 6 Sep 2023 13:20:10 -0400 Subject: [PATCH 4/5] Pin Go version in Travis config Signed-off-by: Byron Ruth --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6607183d..f8d5d148 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ vm: language: go go: - - "1.20.x" + - "1.20.8" go_import_path: github.com/nats-io/nats-server From e1574eca3e79e3f8a80b7ca3cf4cac4a2f619e7e Mon Sep 17 00:00:00 2001 From: Waldemar Quevedo Date: Wed, 6 Sep 2023 11:51:38 -0700 Subject: [PATCH 5/5] Revert "Enables 0s deduplication window duration when the stream has sources (#4476)" This reverts commit db96238ad9b18c9930e63c404d1ae288f8855374. --- server/stream.go | 62 ++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/server/stream.go b/server/stream.go index bfe06b23..fb792845 100644 --- a/server/stream.go +++ b/server/stream.go @@ -805,32 +805,30 @@ func (mset *stream) rebuildDedupe() { mset.ddloaded = true - if mset.cfg.Duplicates > time.Duration(0) { - // We have some messages. Lookup starting sequence by duplicate time window. - sseq := mset.store.GetSeqFromTime(time.Now().Add(-mset.cfg.Duplicates)) - if sseq == 0 { - return + // We have some messages. Lookup starting sequence by duplicate time window. + sseq := mset.store.GetSeqFromTime(time.Now().Add(-mset.cfg.Duplicates)) + if sseq == 0 { + return + } + + var smv StoreMsg + var state StreamState + mset.store.FastState(&state) + + for seq := sseq; seq <= state.LastSeq; seq++ { + sm, err := mset.store.LoadMsg(seq, &smv) + if err != nil { + continue } - - var smv StoreMsg - var state StreamState - mset.store.FastState(&state) - - for seq := sseq; seq <= state.LastSeq; seq++ { - sm, err := mset.store.LoadMsg(seq, &smv) - if err != nil { - continue - } - var msgId string - if len(sm.hdr) > 0 { - if msgId = getMsgId(sm.hdr); msgId != _EMPTY_ { - mset.storeMsgIdLocked(&ddentry{msgId, sm.seq, sm.ts}) - } - } - if seq == state.LastSeq { - mset.lmsgId = msgId + var msgId string + if len(sm.hdr) > 0 { + if msgId = getMsgId(sm.hdr); msgId != _EMPTY_ { + mset.storeMsgIdLocked(&ddentry{msgId, sm.seq, sm.ts}) } } + if seq == state.LastSeq { + mset.lmsgId = msgId + } } } @@ -1025,7 +1023,7 @@ func (s *Server) checkStreamCfg(config *StreamConfig, acc *Account) (StreamConfi if cfg.MaxConsumers == 0 { cfg.MaxConsumers = -1 } - if cfg.Duplicates == 0 && cfg.Mirror == nil && len(cfg.Sources) == 0 { + if cfg.Duplicates == 0 && cfg.Mirror == nil { maxWindow := StreamDefaultDuplicatesWindow if lim.Duplicates > 0 && maxWindow > lim.Duplicates { maxWindow = lim.Duplicates @@ -3488,15 +3486,13 @@ func (mset *stream) storeMsgId(dde *ddentry) { // storeMsgIdLocked will store the message id for duplicate detection. // Lock should he held. func (mset *stream) storeMsgIdLocked(dde *ddentry) { - if mset.cfg.Duplicates > time.Duration(0) { - if mset.ddmap == nil { - mset.ddmap = make(map[string]*ddentry) - } - mset.ddmap[dde.id] = dde - mset.ddarr = append(mset.ddarr, dde) - if mset.ddtmr == nil { - mset.ddtmr = time.AfterFunc(mset.cfg.Duplicates, mset.purgeMsgIds) - } + if mset.ddmap == nil { + mset.ddmap = make(map[string]*ddentry) + } + mset.ddmap[dde.id] = dde + mset.ddarr = append(mset.ddarr, dde) + if mset.ddtmr == nil { + mset.ddtmr = time.AfterFunc(mset.cfg.Duplicates, mset.purgeMsgIds) } }