Added code to constrain size of WAL under most scenarios.

Signed-off-by: Derek Collison <derek@nats.io>

server/const.go

@@ -40,7 +40,7 @@ var (
 
 const (
 	// VERSION is the current version for the server.
-	VERSION = "2.2.0-RC.6"
+	VERSION = "2.2.0-RC.6.1"
 
 	// PROTO is the currently supported protocol.
 	// 0 was the original

server/filestore.go

@@ -2402,6 +2402,17 @@ func (fs *fileStore) LoadMsg(seq uint64) (string, []byte, []byte, int64, error)
 	return "", nil, nil, 0, err
 }
 
+// FastState will fill in state with only the following.
+// Msgs, Bytes, FirstSeq, LastSeq
+func (fs *fileStore) FastState(state *StreamState) {
+	fs.mu.RLock()
+	state.Msgs = fs.state.Msgs
+	state.Bytes = fs.state.Bytes
+	state.FirstSeq = fs.state.FirstSeq
+	state.LastSeq = fs.state.LastSeq
+	fs.mu.RUnlock()
+}
+
 // State returns the current state of the stream.
 func (fs *fileStore) State() StreamState {
 	fs.mu.RLock()

server/jetstream_cluster.go

@@ -755,12 +755,12 @@ func (js *jetStream) monitorCluster() {
 			}
 			// FIXME(dlc) - Deal with errors.
 			if _, didRemoval, err := js.applyMetaEntries(ce.Entries, isRecovering); err == nil {
-				n.Applied(ce.Index)
+				_, nb := n.Applied(ce.Index)
 				if js.hasPeerEntries(ce.Entries) || (didRemoval && time.Since(lastSnapTime) > 2*time.Second) {
 					// Since we received one make sure we have our own since we do not store
 					// our meta state outside of raft.
 					doSnapshot()
-				} else if _, b := n.Size(); b > uint64(len(lastSnap)*4) {
+				} else if nb > uint64(len(lastSnap)*4) {
 					doSnapshot()
 				}
 			}
@@ -1273,7 +1273,6 @@ func (js *jetStream) monitorStream(mset *stream, sa *streamAssignment) {
 	}
 
 	var lastSnap []byte
-	var lastApplied uint64
 
 	// Should only to be called from leader.
 	doSnapshot := func() {
@@ -1283,7 +1282,6 @@ func (js *jetStream) monitorStream(mset *stream, sa *streamAssignment) {
 		if snap := mset.stateSnapshot(); !bytes.Equal(lastSnap, snap) {
 			if err := n.InstallSnapshot(snap); err == nil {
 				lastSnap = snap
-				_, _, lastApplied = n.Progress()
 			}
 		}
 	}
@@ -1313,10 +1311,9 @@ func (js *jetStream) monitorStream(mset *stream, sa *streamAssignment) {
 			// Apply our entries.
 			//TODO mset may be nil see doSnapshot(). applyStreamEntries is sensitive to this
 			if err := js.applyStreamEntries(mset, ce, isRecovering); err == nil {
-				n.Applied(ce.Index)
-				ne := ce.Index - lastApplied
+				ne, nb := n.Applied(ce.Index)
 				// If we have at least min entries to compact, go ahead and snapshot/compact.
-				if ne >= compactNumMin {
+				if ne >= compactNumMin || nb > compactSizeMin {
 					doSnapshot()
 				}
 			} else {
@@ -2538,6 +2535,7 @@ func (js *jetStream) monitorConsumer(o *consumer, ca *consumerAssignment) {
 
 	const (
 		compactInterval = 2 * time.Minute
+		compactSizeMin  = 8 * 1024 * 1024
 		compactNumMin   = 8192
 	)
 
@@ -2545,7 +2543,6 @@ func (js *jetStream) monitorConsumer(o *consumer, ca *consumerAssignment) {
 	defer t.Stop()
 
 	var lastSnap []byte
-	var lastApplied uint64
 
 	// Should only to be called from leader.
 	doSnapshot := func() {
@@ -2553,7 +2550,6 @@ func (js *jetStream) monitorConsumer(o *consumer, ca *consumerAssignment) {
 			if snap := encodeConsumerState(state); !bytes.Equal(lastSnap, snap) {
 				if err := n.InstallSnapshot(snap); err == nil {
 					lastSnap = snap
-					_, _, lastApplied = n.Progress()
 				}
 			}
 		}
@@ -2574,10 +2570,9 @@ func (js *jetStream) monitorConsumer(o *consumer, ca *consumerAssignment) {
 				continue
 			}
 			if err := js.applyConsumerEntries(o, ce, isLeader); err == nil {
-				n.Applied(ce.Index)
-				ne := ce.Index - lastApplied
+				ne, nb := n.Applied(ce.Index)
 				// If we have at least min entries to compact, go ahead and snapshot/compact.
-				if ne >= compactNumMin {
+				if ne >= compactNumMin || nb > compactNumMin {
 					doSnapshot()
 				}
 			} else {

server/memstore.go

@@ -499,6 +499,17 @@ func (ms *memStore) removeMsg(seq uint64, secure bool) bool {
 	return ok
 }
 
+// FastState will fill in state with only the following.
+// Msgs, Bytes, FirstSeq, LastSeq
+func (ms *memStore) FastState(state *StreamState) {
+	ms.mu.RLock()
+	state.Msgs = ms.state.Msgs
+	state.Bytes = ms.state.Bytes
+	state.FirstSeq = ms.state.FirstSeq
+	state.LastSeq = ms.state.LastSeq
+	ms.mu.RUnlock()
+}
+
 func (ms *memStore) State() StreamState {
 	ms.mu.RLock()
 	state := ms.state

server/raft.go

@@ -38,7 +38,7 @@ type RaftNode interface {
 	ForwardProposal(entry []byte) error
 	InstallSnapshot(snap []byte) error
 	SendSnapshot(snap []byte) error
-	Applied(index uint64)
+	Applied(index uint64) (entries uint64, bytes uint64)
 	Compact(index uint64) error
 	State() RaftState
 	Size() (entries, bytes uint64)
@@ -72,6 +72,7 @@ type WAL interface {
 	Purge() (uint64, error)
 	Truncate(seq uint64) error
 	State() StreamState
+	FastState(*StreamState)
 	Stop() error
 	Delete() error
 }
@@ -376,7 +377,9 @@ func (s *Server) startRaftNode(cfg *RaftConfig) (RaftNode, error) {
 		n.setupLastSnapshot()
 	}
 
-	if state := n.wal.State(); state.Msgs > 0 {
+	var state StreamState
+	n.wal.FastState(&state)
+	if state.Msgs > 0 {
 		// TODO(dlc) - Recover our state here.
 		if first, err := n.loadFirstEntry(); err == nil {
 			n.pterm, n.pindex = first.pterm, first.pindex
@@ -683,13 +686,20 @@ func (n *raft) Compact(index uint64) error {
 }
 
 // Applied is to be called when the FSM has applied the committed entries.
-func (n *raft) Applied(index uint64) {
+// Applied will return the number of entries and an estimation of the
+// byte size that could be removed with a snapshot/compact.
+func (n *raft) Applied(index uint64) (entries uint64, bytes uint64) {
 	n.Lock()
 	// Ignore if already applied.
 	if index > n.applied {
 		n.applied = index
 	}
+	var state StreamState
+	n.wal.FastState(&state)
+	entries = n.applied - state.FirstSeq
+	bytes = entries * state.Bytes / state.Msgs
 	n.Unlock()
+	return entries, bytes
 }
 
 // For capturing data needed by snapshot.
@@ -753,7 +763,9 @@ func (n *raft) InstallSnapshot(data []byte) error {
 		return werr
 	}
 
-	if state := n.wal.State(); state.FirstSeq >= n.applied {
+	var state StreamState
+	n.wal.FastState(&state)
+	if state.FirstSeq >= n.applied {
 		n.Unlock()
 		return nil
 	}
@@ -1084,7 +1096,8 @@ func (n *raft) Progress() (index, commit, applied uint64) {
 // Size returns number of entries and total bytes for our WAL.
 func (n *raft) Size() (uint64, uint64) {
 	n.RLock()
-	state := n.wal.State()
+	var state StreamState
+	n.wal.FastState(&state)
 	n.RUnlock()
 	return state.Msgs, state.Bytes
 }
@@ -1669,7 +1682,9 @@ func (n *raft) currentTerm() uint64 {
 
 // Lock should be held.
 func (n *raft) loadFirstEntry() (ae *appendEntry, err error) {
-	return n.loadEntry(n.wal.State().FirstSeq)
+	var state StreamState
+	n.wal.FastState(&state)
+	return n.loadEntry(state.FirstSeq)
 }
 
 func (n *raft) runCatchup(ar *appendEntryResponse, indexUpdatesC <-chan uint64) {
@@ -1795,7 +1810,8 @@ func (n *raft) catchupFollower(ar *appendEntryResponse) {
 	}
 	// Check to make sure we have this entry.
 	start := ar.index + 1
-	state := n.wal.State()
+	var state StreamState
+	n.wal.FastState(&state)
 
 	if start < state.FirstSeq {
 		n.debug("Need to send snapshot to follower")

server/store.go

@@ -76,6 +76,7 @@ type StreamStore interface {
 	Truncate(seq uint64) error
 	GetSeqFromTime(t time.Time) uint64
 	State() StreamState
+	FastState(*StreamState)
 	RegisterStorageUpdates(StorageUpdateHandler)
 	UpdateConfig(cfg *StreamConfig) error
 	Delete() error