Add support for keys with ttl (#177)

* ttl support first commit

* imports fix

* put api args correction

* put options added

* upgrade method added

* upgrade log added

* v0 to v1 migration script added

* error assertion added

* temp migration dir fix

Co-authored-by: yash <yash.chandra@grabpay.com>

internal/data/codec/decoder.go

@@ -3,6 +3,7 @@ package codec
 import (
 	"encoding/binary"
 	"io"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/prologic/bitcask/internal"
@@ -49,13 +50,13 @@ func (d *Decoder) Decode(v *internal.Entry) (int64, error) {
 		return 0, err
 	}
 
-	buf := make([]byte, uint64(actualKeySize)+actualValueSize+checksumSize)
+	buf := make([]byte, uint64(actualKeySize)+actualValueSize+checksumSize+ttlSize)
 	if _, err = io.ReadFull(d.r, buf); err != nil {
 		return 0, errTruncatedData
 	}
 
 	decodeWithoutPrefix(buf, actualKeySize, v)
-	return int64(keySize + valueSize + uint64(actualKeySize) + actualValueSize + checksumSize), nil
+	return int64(keySize + valueSize + uint64(actualKeySize) + actualValueSize + checksumSize + ttlSize), nil
 }
 
 // DecodeEntry decodes a serialized entry
@@ -84,8 +85,18 @@ func getKeyValueSizes(buf []byte, maxKeySize uint32, maxValueSize uint64) (uint3
 
 func decodeWithoutPrefix(buf []byte, valueOffset uint32, v *internal.Entry) {
 	v.Key = buf[:valueOffset]
-	v.Value = buf[valueOffset : len(buf)-checksumSize]
-	v.Checksum = binary.BigEndian.Uint32(buf[len(buf)-checksumSize:])
+	v.Value = buf[valueOffset : len(buf)-checksumSize-ttlSize]
+	v.Checksum = binary.BigEndian.Uint32(buf[len(buf)-checksumSize-ttlSize : len(buf)-ttlSize])
+	v.Expiry = getKeyExpiry(buf)
+}
+
+func getKeyExpiry(buf []byte) *time.Time {
+	expiry := binary.BigEndian.Uint64(buf[len(buf)-ttlSize:])
+	if expiry == uint64(0) {
+		return nil
+	}
+	t := time.Unix(int64(expiry), 0).UTC()
+	return &t
 }
 
 // IsCorruptedData indicates if the error correspondes to possible data corruption

internal/data/codec/decoder_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/binary"
 	"io"
 	"testing"
+	"time"
 
 	"github.com/prologic/bitcask/internal"
 	"github.com/stretchr/testify/assert"
@@ -107,3 +108,23 @@ func TestTruncatedData(t *testing.T) {
 		})
 	}
 }
+
+func TestDecodeWithoutPrefix(t *testing.T) {
+	assert := assert.New(t)
+	e := internal.Entry{}
+	buf := []byte{0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 7, 109, 121, 107, 101, 121, 109, 121, 118, 97, 108, 117, 101, 0, 6, 81, 189, 0, 0, 0, 0, 95, 117, 28, 0}
+	valueOffset := uint32(5)
+	mockTime := time.Date(2020, 10, 1, 0, 0, 0, 0, time.UTC)
+	expectedEntry := internal.Entry{
+		Key:      []byte("mykey"),
+		Value:    []byte("myvalue"),
+		Checksum: 414141,
+		Expiry:   &mockTime,
+	}
+	decodeWithoutPrefix(buf[keySize+valueSize:], valueOffset, &e)
+	assert.Equal(expectedEntry.Key, e.Key)
+	assert.Equal(expectedEntry.Value, e.Value)
+	assert.Equal(expectedEntry.Checksum, e.Checksum)
+	assert.Equal(expectedEntry.Offset, e.Offset)
+	assert.Equal(*expectedEntry.Expiry, *e.Expiry)
+}

internal/data/codec/encoder.go

@@ -13,7 +13,8 @@ const (
 	keySize      = 4
 	valueSize    = 8
 	checksumSize = 4
-	MetaInfoSize = keySize + valueSize + checksumSize
+	ttlSize      = 8
+	MetaInfoSize = keySize + valueSize + checksumSize + ttlSize
 )
 
 // NewEncoder creates a streaming Entry encoder.
@@ -50,9 +51,19 @@ func (e *Encoder) Encode(msg internal.Entry) (int64, error) {
 		return 0, errors.Wrap(err, "failed writing checksum data")
 	}
 
+	bufTTL := bufKeyValue[:ttlSize]
+	if msg.Expiry == nil {
+		binary.BigEndian.PutUint64(bufTTL, uint64(0))
+	} else {
+		binary.BigEndian.PutUint64(bufTTL, uint64(msg.Expiry.Unix()))
+	}
+	if _, err := e.w.Write(bufTTL); err != nil {
+		return 0, errors.Wrap(err, "failed writing ttl data")
+	}
+
 	if err := e.w.Flush(); err != nil {
 		return 0, errors.Wrap(err, "failed flushing data")
 	}
 
-	return int64(keySize + valueSize + len(msg.Key) + len(msg.Value) + checksumSize), nil
+	return int64(keySize + valueSize + len(msg.Key) + len(msg.Value) + checksumSize + ttlSize), nil
 }

internal/data/codec/encoder_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/hex"
 	"testing"
+	"time"
 
 	"github.com/prologic/bitcask/internal"
 	"github.com/stretchr/testify/assert"
@@ -14,15 +15,17 @@ func TestEncode(t *testing.T) {
 	assert := assert.New(t)
 
 	var buf bytes.Buffer
+	mockTime := time.Date(2020, 10, 1, 0, 0, 0, 0, time.UTC)
 	encoder := NewEncoder(&buf)
 	_, err := encoder.Encode(internal.Entry{
 		Key:      []byte("mykey"),
 		Value:    []byte("myvalue"),
 		Checksum: 414141,
 		Offset:   424242,
+		Expiry:   &mockTime,
 	})
 
-	expectedHex := "0000000500000000000000076d796b65796d7976616c7565000651bd"
+	expectedHex := "0000000500000000000000076d796b65796d7976616c7565000651bd000000005f751c00"
 	if assert.NoError(err) {
 		assert.Equal(expectedHex, hex.EncodeToString(buf.Bytes()))
 	}