Fix for writing messages after restart and delivery count suppression on max

Signed-off-by: Derek Collison <derek@nats.io>

server/filestore.go

@@ -1,4 +1,4 @@
-// Copyright 2019 The NATS Authors
+// Copyright 2019-2020 The NATS Authors
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -281,7 +281,7 @@ func (fs *fileStore) recoverMsgBlock(fi os.FileInfo, index uint64) *msgBlock {
 	mb.ifn = path.Join(fs.fcfg.StoreDir, msgDir, fmt.Sprintf(indexScan, index))
 
 	// Open up the message file, but we will try to recover from the index file.
-	// We will check that the last checksufs match.
+	// We will check that the last checksums match.
 	file, err := os.Open(mb.mfn)
 	if err != nil {
 		return nil
@@ -373,6 +373,8 @@ func (fs *fileStore) recoverMsgs() error {
 
 	if len(fs.blks) == 0 {
 		_, err = fs.newMsgBlockForWrite()
+	} else {
+		err = fs.enableLastMsgBlockForWriting()
 	}
 
 	return err
@@ -429,6 +431,24 @@ func (fs *fileStore) newMsgBlockForWrite() (*msgBlock, error) {
 	return mb, nil
 }
 
+// Make sure we can write to the last message block.
+// Lock should be held.
+func (fs *fileStore) enableLastMsgBlockForWriting() error {
+	mb := fs.lmb
+	if mb == nil {
+		return fmt.Errorf("No last message block assigned, can not enable for writing")
+	}
+	if mb.mfd != nil {
+		return nil
+	}
+	mfd, err := os.OpenFile(mb.mfn, os.O_APPEND|os.O_CREATE|os.O_RDWR, 0644)
+	if err != nil {
+		return fmt.Errorf("Error opening msg block file [%q]: %v", mb.mfn, err)
+	}
+	mb.mfd = mfd
+	return nil
+}
+
 // Store stores a message.
 func (fs *fileStore) StoreMsg(subj string, msg []byte) (uint64, error) {
 	fs.mu.Lock()

server/filestore_test.go

@@ -1,4 +1,4 @@
-// Copyright 2019 The NATS Authors
+// Copyright 2019-2020 The NATS Authors
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -94,7 +94,6 @@ func TestFileStoreBasicWriteMsgsAndRestore(t *testing.T) {
 
 	// Write 100 msgs
 	toStore := uint64(100)
-
 	for i := uint64(1); i <= toStore; i++ {
 		msg := []byte(fmt.Sprintf("[%08d] Hello World!", i))
 		if seq, err := fs.StoreMsg(subj, msg); err != nil {
@@ -116,6 +115,7 @@ func TestFileStoreBasicWriteMsgsAndRestore(t *testing.T) {
 	// Stop will flush to disk.
 	fs.Stop()
 
+	// Restart
 	fs, err = newFileStore(fcfg, MsgSetConfig{Name: "dlc", Storage: FileStorage})
 	if err != nil {
 		t.Fatalf("Unexpected error: %v", err)
@@ -129,6 +129,39 @@ func TestFileStoreBasicWriteMsgsAndRestore(t *testing.T) {
 	if stats.Bytes != expectedSize {
 		t.Fatalf("Expected %d bytes, got %d", expectedSize, stats.Bytes)
 	}
+
+	// Now write 100 more msgs
+	for i := uint64(101); i <= toStore*2; i++ {
+		msg := []byte(fmt.Sprintf("[%08d] Hello World!", i))
+		if seq, err := fs.StoreMsg(subj, msg); err != nil {
+			t.Fatalf("Error storing msg: %v", err)
+		} else if seq != uint64(i) {
+			t.Fatalf("Expected sequence to be %d, got %d", i, seq)
+		}
+	}
+	stats = fs.Stats()
+	if stats.Msgs != toStore*2 {
+		t.Fatalf("Expected %d msgs, got %d", toStore*2, stats.Msgs)
+	}
+
+	// Now cycle again and make sure that last batch was stored.
+	// Stop will flush to disk.
+	fs.Stop()
+
+	// Restart
+	fs, err = newFileStore(fcfg, MsgSetConfig{Name: "dlc", Storage: FileStorage})
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer fs.Stop()
+
+	stats = fs.Stats()
+	if stats.Msgs != toStore*2 {
+		t.Fatalf("Expected %d msgs, got %d", toStore*2, stats.Msgs)
+	}
+	if stats.Bytes != expectedSize*2 {
+		t.Fatalf("Expected %d bytes, got %d", expectedSize*2, stats.Bytes)
+	}
 }
 
 func TestFileStoreMsgLimit(t *testing.T) {

server/msgset.go

@@ -1,4 +1,4 @@
-// Copyright 2019 The NATS Authors
+// Copyright 2019-2020 The NATS Authors
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at

server/observable.go

@@ -821,6 +821,8 @@ func (o *Observable) getNextMsg() (string, []byte, uint64, uint64, error) {
 			o.rdq = append(o.rdq[:0], o.rdq[1:]...)
 			dcount = o.incDeliveryCount(seq)
 			if o.maxdc > 0 && dcount > o.maxdc {
+				// Make sure to remove from pending.
+				delete(o.pending, seq)
 				continue
 			}
 		}