Add in support for segmented binary stream snapshots.

Streams with many interior deletes was causing issues due to the fact that the interior deletes were represented as a sorted []uint64.
This approach introduces 3 sub types of delete blocks, avl bitmask tree, a run length encoding, and the legacy format above.
We also take into account large interior deletes such that on receiving a snapshot we can skip things we already know about.

Signed-off-by: Derek Collison <derek@nats.io>
This commit is contained in:
Derek Collison
2023-06-30 22:49:48 -07:00
parent cf393140ab
commit 4d7cd26956
13 changed files with 887 additions and 77 deletions

View File

@@ -4519,3 +4519,25 @@ func TestJetStreamClusterSnapshotAndRestoreWithHealthz(t *testing.T) {
require_NoError(t, err)
require_True(t, si.State.Msgs == uint64(toSend))
}
func TestJetStreamBinaryStreamSnapshotCapability(t *testing.T) {
c := createJetStreamClusterExplicit(t, "NATS", 3)
defer c.shutdown()
nc, js := jsClientConnect(t, c.randomServer())
defer nc.Close()
_, err := js.AddStream(&nats.StreamConfig{
Name: "TEST",
Subjects: []string{"foo"},
Replicas: 3,
})
require_NoError(t, err)
mset, err := c.streamLeader(globalAccountName, "TEST").GlobalAccount().lookupStream("TEST")
require_NoError(t, err)
if !mset.supportsBinarySnapshot() {
t.Fatalf("Expected to signal that we could support binary stream snapshots")
}
}