From bf49f23bb1b9b3d0bbfc31315d5d86ba13081ce3 Mon Sep 17 00:00:00 2001
From: Derek Collison <derek@nats.io>
Date: Sat, 28 Jan 2023 11:34:55 -0800
Subject: [PATCH] Only hold on to so many pending in memory, will fetch from
 WAL

Signed-off-by: Derek Collison <derek@nats.io>
---
 server/raft.go | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/server/raft.go b/server/raft.go
index 1c95aeef..cec53ab5 100644
--- a/server/raft.go
+++ b/server/raft.go
@@ -2987,11 +2987,15 @@ func (n *raft) processAppendEntry(ae *appendEntry, sub *subscription) {
 				return
 			}
 			// Save in memory for faster processing during applyCommit.
-			n.pae[n.pindex] = ae
-			if l := len(n.pae); l > paeWarnThreshold && l%paeWarnModulo == 0 {
-				n.warn("%d append entries pending", len(n.pae))
+			// Only save so many however to avoid memory bloat.
+			if l := len(n.pae); l <= paeDropThreshold {
+				n.pae[n.pindex], l = ae, l+1
+				if l > paeWarnThreshold && l%paeWarnModulo == 0 {
+					n.warn("%d append entries pending", len(n.pae))
+				}
+			} else {
+				n.debug("Not saving to append entries pending")
 			}
-
 		} else {
 			// This is a replay on startup so just take the appendEntry version.
 			n.pterm = ae.term
@@ -3140,8 +3144,11 @@ func (n *raft) storeToWAL(ae *appendEntry) error {
 	return nil
 }
 
-const paeWarnThreshold = 20_000
-const paeWarnModulo = 5_000
+const (
+	paeDropThreshold = 20_000
+	paeWarnThreshold = 10_000
+	paeWarnModulo    = 5_000
+)
 
 func (n *raft) sendAppendEntry(entries []*Entry) {
 	n.Lock()