Purge api added to remove expired keys (#204)

* purge api added

* merged with master, import order fix

* purge api renamed to RunGC

Co-authored-by: yash <yash.chandra@grabpay.com>

internal/index/ttl_index.go

@@ -0,0 +1,71 @@
+package index
+
+import (
+	"encoding/binary"
+	"io"
+	"os"
+	"time"
+
+	art "github.com/plar/go-adaptive-radix-tree"
+	"github.com/prologic/bitcask/internal"
+)
+
+type ttlIndexer struct{}
+
+func NewTTLIndexer() Indexer {
+	return ttlIndexer{}
+}
+
+func (i ttlIndexer) Save(t art.Tree, path string) error {
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	buf := make([]byte, int64Size)
+	for it := t.Iterator(); it.HasNext(); {
+		node, err := it.Next()
+		if err != nil {
+			return err
+		}
+		// save key
+		err = writeBytes(node.Key(), f)
+		if err != nil {
+			return err
+		}
+		// save key ttl
+		binary.BigEndian.PutUint64(buf, uint64(node.Value().(time.Time).Unix()))
+		_, err = f.Write(buf)
+		if err != nil {
+			return err
+		}
+	}
+	return f.Sync()
+}
+
+func (i ttlIndexer) Load(path string, maxKeySize uint32) (art.Tree, bool, error) {
+	t := art.New()
+	if !internal.Exists(path) {
+		return t, false, nil
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return t, true, err
+	}
+	buf := make([]byte, int64Size)
+	for {
+		key, err := readKeyBytes(f, maxKeySize)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return t, true, err
+		}
+		_, err = io.ReadFull(f, buf)
+		if err != nil {
+			return t, true, err
+		}
+		expiry := time.Unix(int64(binary.BigEndian.Uint64(buf)), 0).UTC()
+		t.Insert(key, expiry)
+	}
+	return t, true, nil
+}

internal/index/ttl_index_test.go

@@ -0,0 +1,54 @@
+package index
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	art "github.com/plar/go-adaptive-radix-tree"
+	assert2 "github.com/stretchr/testify/assert"
+)
+
+func Test_TTLIndexer(t *testing.T) {
+	assert := assert2.New(t)
+	tempDir, err := ioutil.TempDir("", "bitcask")
+	assert.NoError(err)
+	defer os.RemoveAll(tempDir)
+
+	currTime := time.Date(2020, 12, 27, 0, 0, 0, 0, time.UTC)
+	trie := art.New()
+
+	t.Run("LoadEmpty", func(t *testing.T) {
+		newTrie, found, err := NewTTLIndexer().Load(filepath.Join(tempDir, "ttl_index"), 4)
+		assert.NoError(err)
+		assert.False(found)
+		assert.Equal(trie, newTrie)
+	})
+
+	t.Run("Save", func(t *testing.T) {
+		trie.Insert([]byte("key"), currTime)
+		err := NewTTLIndexer().Save(trie, filepath.Join(tempDir, "ttl_index"))
+		assert.NoError(err)
+		trie.Insert([]byte("foo"), currTime.Add(24*time.Hour))
+		err = NewTTLIndexer().Save(trie, filepath.Join(tempDir, "ttl_index"))
+		assert.NoError(err)
+		trie.Insert([]byte("key"), currTime.Add(-24*time.Hour))
+		err = NewTTLIndexer().Save(trie, filepath.Join(tempDir, "ttl_index"))
+		assert.NoError(err)
+	})
+
+	t.Run("Load", func(t *testing.T) {
+		newTrie, found, err := NewTTLIndexer().Load(filepath.Join(tempDir, "ttl_index"), 4)
+		assert.NoError(err)
+		assert.True(found)
+		assert.Equal(2, newTrie.Size())
+		value, found := newTrie.Search([]byte("key"))
+		assert.True(found)
+		assert.Equal(currTime.Add(-24*time.Hour), value)
+		value, found = newTrie.Search([]byte("foo"))
+		assert.True(found)
+		assert.Equal(currTime.Add(24*time.Hour), value)
+	})
+}