From b517229c3299e684a15398abbdc945bd0530e620 Mon Sep 17 00:00:00 2001
From: Derek Collison <derek@nats.io>
Date: Fri, 6 Aug 2021 14:44:00 -0700
Subject: [PATCH] Fix for #2417

Signed-off-by: Derek Collison <derek@nats.io>
---
 server/filestore.go      | 14 +++++++++++++-
 server/filestore_test.go | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/server/filestore.go b/server/filestore.go
index 30f04d87..4d4d3331 100644
--- a/server/filestore.go
+++ b/server/filestore.go
@@ -4145,7 +4145,19 @@ func (fs *fileStore) Delete() error {
 	if err := fs.Stop(); err != nil {
 		return err
 	}
-	return os.RemoveAll(fs.fcfg.StoreDir)
+
+	err := os.RemoveAll(fs.fcfg.StoreDir)
+	if err == nil {
+		return nil
+	}
+	ttl := time.Now().Add(time.Second)
+	for time.Now().Before(ttl) {
+		time.Sleep(10 * time.Millisecond)
+		if err = os.RemoveAll(fs.fcfg.StoreDir); err == nil {
+			return nil
+		}
+	}
+	return err
 }
 
 // Lock should be held.
diff --git a/server/filestore_test.go b/server/filestore_test.go
index 5a59506f..79d2760a 100644
--- a/server/filestore_test.go
+++ b/server/filestore_test.go
@@ -2675,6 +2675,41 @@ func TestFileStoreStreamStateDeleted(t *testing.T) {
 	}
 }
 
+// We have reports that sometimes under load a stream could complain about a storage directory
+// not being empty.
+func TestFileStoreStreamDeleteDirNotEmpty(t *testing.T) {
+	storeDir := createDir(t, JetStreamStoreDir)
+	defer removeDir(t, storeDir)
+
+	fs, err := newFileStore(FileStoreConfig{StoreDir: storeDir}, StreamConfig{Name: "zzz", Storage: FileStorage})
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer fs.Stop()
+
+	subj, toStore := "foo", uint64(10)
+	for i := uint64(1); i <= toStore; i++ {
+		msg := []byte(fmt.Sprintf("[%08d] Hello World!", i))
+		if _, _, err := fs.StoreMsg(subj, nil, msg); err != nil {
+			t.Fatalf("Error storing msg: %v", err)
+		}
+	}
+
+	ready := make(chan bool)
+	go func() {
+		g := path.Join(storeDir, "g")
+		ready <- true
+		for i := 0; i < 100; i++ {
+			ioutil.WriteFile(g, []byte("OK"), defaultFilePerms)
+		}
+	}()
+
+	<-ready
+	if err := fs.Delete(); err != nil {
+		t.Fatalf("Delete returned an error: %v", err)
+	}
+}
+
 func TestFileStoreConsumerPerf(t *testing.T) {
 	// Comment out to run, holding place for now.
 	t.SkipNow()