Add all files again with v2 postfix to URL

v2/internal/config/config.go

@@ -0,0 +1,51 @@
+package config
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+)
+
+// Config contains the bitcask configuration parameters
+type Config struct {
+	MaxDatafileSize         int    `json:"max_datafile_size"`
+	MaxKeySize              uint32 `json:"max_key_size"`
+	MaxValueSize            uint64 `json:"max_value_size"`
+	Sync                    bool   `json:"sync"`
+	AutoRecovery            bool   `json:"autorecovery"`
+	DBVersion               uint32 `json:"db_version"`
+	DirFileModeBeforeUmask  os.FileMode
+	FileFileModeBeforeUmask os.FileMode
+}
+
+// Load loads a configuration from the given path
+func Load(path string) (*Config, error) {
+	var cfg Config
+
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := json.Unmarshal(data, &cfg); err != nil {
+		return nil, err
+	}
+
+	return &cfg, nil
+}
+
+// Save saves the configuration to the provided path
+func (c *Config) Save(path string) error {
+
+	data, err := json.Marshal(c)
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(path, data, c.FileFileModeBeforeUmask)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

v2/internal/data/codec/decoder.go

@@ -0,0 +1,110 @@
+package codec
+
+import (
+	"encoding/binary"
+	"io"
+	"time"
+
+	"git.mills.io/prologic/bitcask/v2/internal"
+	"github.com/pkg/errors"
+)
+
+var (
+	errInvalidKeyOrValueSize = errors.New("key/value size is invalid")
+	errCantDecodeOnNilEntry  = errors.New("can't decode on nil entry")
+	errTruncatedData         = errors.New("data is truncated")
+)
+
+// NewDecoder creates a streaming Entry decoder.
+func NewDecoder(r io.Reader, maxKeySize uint32, maxValueSize uint64) *Decoder {
+	return &Decoder{
+		r:            r,
+		maxKeySize:   maxKeySize,
+		maxValueSize: maxValueSize,
+	}
+}
+
+// Decoder wraps an underlying io.Reader and allows you to stream
+// Entry decodings on it.
+type Decoder struct {
+	r            io.Reader
+	maxKeySize   uint32
+	maxValueSize uint64
+}
+
+// Decode decodes the next Entry from the current stream
+func (d *Decoder) Decode(v *internal.Entry) (int64, error) {
+	if v == nil {
+		return 0, errCantDecodeOnNilEntry
+	}
+
+	prefixBuf := make([]byte, keySize+valueSize)
+
+	_, err := io.ReadFull(d.r, prefixBuf)
+	if err != nil {
+		return 0, err
+	}
+
+	actualKeySize, actualValueSize, err := getKeyValueSizes(prefixBuf, d.maxKeySize, d.maxValueSize)
+	if err != nil {
+		return 0, err
+	}
+
+	buf := make([]byte, uint64(actualKeySize)+actualValueSize+checksumSize+ttlSize)
+	if _, err = io.ReadFull(d.r, buf); err != nil {
+		return 0, errTruncatedData
+	}
+
+	decodeWithoutPrefix(buf, actualKeySize, v)
+	return int64(keySize + valueSize + uint64(actualKeySize) + actualValueSize + checksumSize + ttlSize), nil
+}
+
+// DecodeEntry decodes a serialized entry
+func DecodeEntry(b []byte, e *internal.Entry, maxKeySize uint32, maxValueSize uint64) error {
+	valueOffset, _, err := getKeyValueSizes(b, maxKeySize, maxValueSize)
+	if err != nil {
+		return errors.Wrap(err, "key/value sizes are invalid")
+	}
+
+	decodeWithoutPrefix(b[keySize+valueSize:], valueOffset, e)
+
+	return nil
+}
+
+func getKeyValueSizes(buf []byte, maxKeySize uint32, maxValueSize uint64) (uint32, uint64, error) {
+	actualKeySize := binary.BigEndian.Uint32(buf[:keySize])
+	actualValueSize := binary.BigEndian.Uint64(buf[keySize:])
+
+	if (maxKeySize > 0 && actualKeySize > maxKeySize) || (maxValueSize > 0 && actualValueSize > maxValueSize) || actualKeySize == 0 {
+
+		return 0, 0, errInvalidKeyOrValueSize
+	}
+
+	return actualKeySize, actualValueSize, nil
+}
+
+func decodeWithoutPrefix(buf []byte, valueOffset uint32, v *internal.Entry) {
+	v.Key = buf[:valueOffset]
+	v.Value = buf[valueOffset : len(buf)-checksumSize-ttlSize]
+	v.Checksum = binary.BigEndian.Uint32(buf[len(buf)-checksumSize-ttlSize : len(buf)-ttlSize])
+	v.Expiry = getKeyExpiry(buf)
+}
+
+func getKeyExpiry(buf []byte) *time.Time {
+	expiry := binary.BigEndian.Uint64(buf[len(buf)-ttlSize:])
+	if expiry == uint64(0) {
+		return nil
+	}
+	t := time.Unix(int64(expiry), 0).UTC()
+	return &t
+}
+
+// IsCorruptedData indicates if the error correspondes to possible data corruption
+func IsCorruptedData(err error) bool {
+	switch err {
+	case errCantDecodeOnNilEntry, errInvalidKeyOrValueSize, errTruncatedData:
+		return true
+	default:
+		return false
+	}
+}

v2/internal/data/codec/decoder_test.go

@@ -0,0 +1,130 @@
+package codec
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+	"testing"
+	"time"
+
+	"git.mills.io/prologic/bitcask/v2/internal"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDecodeOnNilEntry(t *testing.T) {
+	t.Parallel()
+	assert := assert.New(t)
+	decoder := NewDecoder(&bytes.Buffer{}, 1, 1)
+
+	_, err := decoder.Decode(nil)
+	if assert.Error(err) {
+		assert.Equal(errCantDecodeOnNilEntry, err)
+	}
+}
+
+func TestShortPrefix(t *testing.T) {
+	t.Parallel()
+	assert := assert.New(t)
+	maxKeySize, maxValueSize := uint32(10), uint64(20)
+	prefix := make([]byte, keySize+valueSize)
+	binary.BigEndian.PutUint32(prefix, 1)
+	binary.BigEndian.PutUint64(prefix[keySize:], 1)
+
+	truncBytesCount := 2
+	buf := bytes.NewBuffer(prefix[:keySize+valueSize-truncBytesCount])
+	decoder := NewDecoder(buf, maxKeySize, maxValueSize)
+	_, err := decoder.Decode(&internal.Entry{})
+	if assert.Error(err) {
+		assert.Equal(io.ErrUnexpectedEOF, err)
+	}
+}
+
+func TestInvalidValueKeySizes(t *testing.T) {
+	assert := assert.New(t)
+	maxKeySize, maxValueSize := uint32(10), uint64(20)
+
+	tests := []struct {
+		keySize   uint32
+		valueSize uint64
+		name      string
+	}{
+		{keySize: 0, valueSize: 5, name: "zero key size"}, //zero value size is correct for tombstones
+		{keySize: 11, valueSize: 5, name: "key size overflow"},
+		{keySize: 5, valueSize: 21, name: "value size overflow"},
+		{keySize: 11, valueSize: 21, name: "key and value size overflow"},
+	}
+
+	for i := range tests {
+		i := i
+		t.Run(tests[i].name, func(t *testing.T) {
+			t.Parallel()
+			prefix := make([]byte, keySize+valueSize)
+			binary.BigEndian.PutUint32(prefix, tests[i].keySize)
+			binary.BigEndian.PutUint64(prefix[keySize:], tests[i].valueSize)
+
+			buf := bytes.NewBuffer(prefix)
+			decoder := NewDecoder(buf, maxKeySize, maxValueSize)
+			_, err := decoder.Decode(&internal.Entry{})
+			if assert.Error(err) {
+				assert.Equal(errInvalidKeyOrValueSize, err)
+			}
+		})
+	}
+}
+
+func TestTruncatedData(t *testing.T) {
+	assert := assert.New(t)
+	maxKeySize, maxValueSize := uint32(10), uint64(20)
+
+	key := []byte("foo")
+	value := []byte("bar")
+	data := make([]byte, keySize+valueSize+len(key)+len(value)+checksumSize)
+
+	binary.BigEndian.PutUint32(data, uint32(len(key)))
+	binary.BigEndian.PutUint64(data[keySize:], uint64(len(value)))
+	copy(data[keySize+valueSize:], key)
+	copy(data[keySize+valueSize+len(key):], value)
+	copy(data[keySize+valueSize+len(key)+len(value):], bytes.Repeat([]byte("0"), checksumSize))
+
+	tests := []struct {
+		data []byte
+		name string
+	}{
+		{data: data[:keySize+valueSize+len(key)-1], name: "truncated key"},
+		{data: data[:keySize+valueSize+len(key)+len(value)-1], name: "truncated value"},
+		{data: data[:keySize+valueSize+len(key)+len(value)+checksumSize-1], name: "truncated checksum"},
+	}
+
+	for i := range tests {
+		i := i
+		t.Run(tests[i].name, func(t *testing.T) {
+			t.Parallel()
+			buf := bytes.NewBuffer(tests[i].data)
+			decoder := NewDecoder(buf, maxKeySize, maxValueSize)
+			_, err := decoder.Decode(&internal.Entry{})
+			if assert.Error(err) {
+				assert.Equal(errTruncatedData, err)
+			}
+		})
+	}
+}
+
+func TestDecodeWithoutPrefix(t *testing.T) {
+	assert := assert.New(t)
+	e := internal.Entry{}
+	buf := []byte{0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 7, 109, 121, 107, 101, 121, 109, 121, 118, 97, 108, 117, 101, 0, 6, 81, 189, 0, 0, 0, 0, 95, 117, 28, 0}
+	valueOffset := uint32(5)
+	mockTime := time.Date(2020, 10, 1, 0, 0, 0, 0, time.UTC)
+	expectedEntry := internal.Entry{
+		Key:      []byte("mykey"),
+		Value:    []byte("myvalue"),
+		Checksum: 414141,
+		Expiry:   &mockTime,
+	}
+	decodeWithoutPrefix(buf[keySize+valueSize:], valueOffset, &e)
+	assert.Equal(expectedEntry.Key, e.Key)
+	assert.Equal(expectedEntry.Value, e.Value)
+	assert.Equal(expectedEntry.Checksum, e.Checksum)
+	assert.Equal(expectedEntry.Offset, e.Offset)
+	assert.Equal(*expectedEntry.Expiry, *e.Expiry)
+}

v2/internal/data/codec/encoder.go

@@ -0,0 +1,69 @@
+package codec
+
+import (
+	"bufio"
+	"encoding/binary"
+	"io"
+
+	"git.mills.io/prologic/bitcask/v2/internal"
+	"github.com/pkg/errors"
+)
+
+const (
+	keySize      = 4
+	valueSize    = 8
+	checksumSize = 4
+	ttlSize      = 8
+	MetaInfoSize = keySize + valueSize + checksumSize + ttlSize
+)
+
+// NewEncoder creates a streaming Entry encoder.
+func NewEncoder(w io.Writer) *Encoder {
+	return &Encoder{w: bufio.NewWriter(w)}
+}
+
+// Encoder wraps an underlying io.Writer and allows you to stream
+// Entry encodings on it.
+type Encoder struct {
+	w *bufio.Writer
+}
+
+// Encode takes any Entry and streams it to the underlying writer.
+// Messages are framed with a key-length and value-length prefix.
+func (e *Encoder) Encode(msg internal.Entry) (int64, error) {
+	var bufKeyValue = make([]byte, keySize+valueSize)
+	binary.BigEndian.PutUint32(bufKeyValue[:keySize], uint32(len(msg.Key)))
+	binary.BigEndian.PutUint64(bufKeyValue[keySize:keySize+valueSize], uint64(len(msg.Value)))
+	if _, err := e.w.Write(bufKeyValue); err != nil {
+		return 0, errors.Wrap(err, "failed writing key & value length prefix")
+	}
+
+	if _, err := e.w.Write(msg.Key); err != nil {
+		return 0, errors.Wrap(err, "failed writing key data")
+	}
+	if _, err := e.w.Write(msg.Value); err != nil {
+		return 0, errors.Wrap(err, "failed writing value data")
+	}
+
+	bufChecksumSize := bufKeyValue[:checksumSize]
+	binary.BigEndian.PutUint32(bufChecksumSize, msg.Checksum)
+	if _, err := e.w.Write(bufChecksumSize); err != nil {
+		return 0, errors.Wrap(err, "failed writing checksum data")
+	}
+
+	bufTTL := bufKeyValue[:ttlSize]
+	if msg.Expiry == nil {
+		binary.BigEndian.PutUint64(bufTTL, uint64(0))
+	} else {
+		binary.BigEndian.PutUint64(bufTTL, uint64(msg.Expiry.Unix()))
+	}
+	if _, err := e.w.Write(bufTTL); err != nil {
+		return 0, errors.Wrap(err, "failed writing ttl data")
+	}
+
+	if err := e.w.Flush(); err != nil {
+		return 0, errors.Wrap(err, "failed flushing data")
+	}
+
+	return int64(keySize + valueSize + len(msg.Key) + len(msg.Value) + checksumSize + ttlSize), nil
+}

v2/internal/data/codec/encoder_test.go

@@ -0,0 +1,32 @@
+package codec
+
+import (
+	"bytes"
+	"encoding/hex"
+	"testing"
+	"time"
+
+	"git.mills.io/prologic/bitcask/v2/internal"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEncode(t *testing.T) {
+	t.Parallel()
+	assert := assert.New(t)
+
+	var buf bytes.Buffer
+	mockTime := time.Date(2020, 10, 1, 0, 0, 0, 0, time.UTC)
+	encoder := NewEncoder(&buf)
+	_, err := encoder.Encode(internal.Entry{
+		Key:      []byte("mykey"),
+		Value:    []byte("myvalue"),
+		Checksum: 414141,
+		Offset:   424242,
+		Expiry:   &mockTime,
+	})
+
+	expectedHex := "0000000500000000000000076d796b65796d7976616c7565000651bd000000005f751c00"
+	if assert.NoError(err) {
+		assert.Equal(expectedHex, hex.EncodeToString(buf.Bytes()))
+	}
+}

v2/internal/data/datafile.go

@@ -0,0 +1,200 @@
+package data
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"git.mills.io/prologic/bitcask/v2/internal"
+	"git.mills.io/prologic/bitcask/v2/internal/data/codec"
+	"github.com/pkg/errors"
+	"golang.org/x/exp/mmap"
+)
+
+const (
+	defaultDatafileFilename = "%09d.data"
+)
+
+var (
+	errReadonly  = errors.New("error: read only datafile")
+	errReadError = errors.New("error: read error")
+)
+
+// Datafile is an interface  that represents a readable and writeable datafile
+type Datafile interface {
+	FileID() int
+	Name() string
+	Close() error
+	Sync() error
+	Size() int64
+	Read() (internal.Entry, int64, error)
+	ReadAt(index, size int64) (internal.Entry, error)
+	Write(internal.Entry) (int64, int64, error)
+}
+
+type datafile struct {
+	sync.RWMutex
+
+	id           int
+	r            *os.File
+	ra           *mmap.ReaderAt
+	w            *os.File
+	offset       int64
+	dec          *codec.Decoder
+	enc          *codec.Encoder
+	maxKeySize   uint32
+	maxValueSize uint64
+}
+
+// NewDatafile opens an existing datafile
+func NewDatafile(path string, id int, readonly bool, maxKeySize uint32, maxValueSize uint64, fileMode os.FileMode) (Datafile, error) {
+	var (
+		r   *os.File
+		ra  *mmap.ReaderAt
+		w   *os.File
+		err error
+	)
+
+	fn := filepath.Join(path, fmt.Sprintf(defaultDatafileFilename, id))
+
+	if !readonly {
+		w, err = os.OpenFile(fn, os.O_WRONLY|os.O_APPEND|os.O_CREATE, fileMode)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	r, err = os.Open(fn)
+	if err != nil {
+		return nil, err
+	}
+	stat, err := r.Stat()
+	if err != nil {
+		return nil, errors.Wrap(err, "error calling Stat()")
+	}
+
+	if readonly {
+		ra, err = mmap.Open(fn)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	offset := stat.Size()
+
+	dec := codec.NewDecoder(r, maxKeySize, maxValueSize)
+	enc := codec.NewEncoder(w)
+
+	return &datafile{
+		id:           id,
+		r:            r,
+		ra:           ra,
+		w:            w,
+		offset:       offset,
+		dec:          dec,
+		enc:          enc,
+		maxKeySize:   maxKeySize,
+		maxValueSize: maxValueSize,
+	}, nil
+}
+
+func (df *datafile) FileID() int {
+	return df.id
+}
+
+func (df *datafile) Name() string {
+	return df.r.Name()
+}
+
+func (df *datafile) Close() error {
+	defer func() {
+		if df.ra != nil {
+			df.ra.Close()
+		}
+		df.r.Close()
+	}()
+
+	// Readonly datafile -- Nothing further to close on the write side
+	if df.w == nil {
+		return nil
+	}
+
+	err := df.Sync()
+	if err != nil {
+		return err
+	}
+	return df.w.Close()
+}
+
+func (df *datafile) Sync() error {
+	if df.w == nil {
+		return nil
+	}
+	return df.w.Sync()
+}
+
+func (df *datafile) Size() int64 {
+	df.RLock()
+	defer df.RUnlock()
+	return df.offset
+}
+
+// Read reads the next entry from the datafile
+func (df *datafile) Read() (e internal.Entry, n int64, err error) {
+	df.Lock()
+	defer df.Unlock()
+
+	n, err = df.dec.Decode(&e)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// ReadAt the entry located at index offset with expected serialized size
+func (df *datafile) ReadAt(index, size int64) (e internal.Entry, err error) {
+	var n int
+
+	b := make([]byte, size)
+
+	df.RLock()
+	defer df.RUnlock()
+
+	if df.ra != nil {
+		n, err = df.ra.ReadAt(b, index)
+	} else {
+		n, err = df.r.ReadAt(b, index)
+	}
+	if err != nil {
+		return
+	}
+	if int64(n) != size {
+		err = errReadError
+		return
+	}
+
+	codec.DecodeEntry(b, &e, df.maxKeySize, df.maxValueSize)
+
+	return
+}
+
+func (df *datafile) Write(e internal.Entry) (int64, int64, error) {
+	if df.w == nil {
+		return -1, 0, errReadonly
+	}
+
+	df.Lock()
+	defer df.Unlock()
+
+	e.Offset = df.offset
+
+	n, err := df.enc.Encode(e)
+	if err != nil {
+		return -1, 0, err
+	}
+	df.offset += n
+
+	return e.Offset, n, nil
+}

v2/internal/data/recover.go

@@ -0,0 +1,95 @@
+package data
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"git.mills.io/prologic/bitcask/v2/internal"
+	"git.mills.io/prologic/bitcask/v2/internal/config"
+	"git.mills.io/prologic/bitcask/v2/internal/data/codec"
+)
+
+// CheckAndRecover checks and recovers the last datafile.
+// If the datafile isn't corrupted, this is a noop. If it is,
+// the longest non-corrupted prefix will be kept and the rest
+// will be *deleted*. Also, the index file is also *deleted* which
+// will be automatically recreated on next startup.
+func CheckAndRecover(path string, cfg *config.Config) error {
+	dfs, err := internal.GetDatafiles(path)
+	if err != nil {
+		return fmt.Errorf("scanning datafiles: %s", err)
+	}
+	if len(dfs) == 0 {
+		return nil
+	}
+	f := dfs[len(dfs)-1]
+	recovered, err := recoverDatafile(f, cfg)
+	if err != nil {
+		return fmt.Errorf("error recovering data file: %s", err)
+	}
+	if recovered {
+		if err := os.Remove(filepath.Join(path, "index")); err != nil {
+			return fmt.Errorf("error deleting the index on recovery: %s", err)
+		}
+	}
+	return nil
+}
+
+func recoverDatafile(path string, cfg *config.Config) (recovered bool, err error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return false, fmt.Errorf("opening the datafile: %s", err)
+	}
+	defer func() {
+		closeErr := f.Close()
+		if err == nil {
+			err = closeErr
+		}
+	}()
+	dir, file := filepath.Split(path)
+	rPath := filepath.Join(dir, fmt.Sprintf("%s.recovered", file))
+	fr, err := os.OpenFile(rPath, os.O_CREATE|os.O_WRONLY, os.ModePerm)
+	if err != nil {
+		return false, fmt.Errorf("creating the recovered datafile: %w", err)
+	}
+	defer func() {
+		closeErr := fr.Close()
+		if err == nil {
+			err = closeErr
+		}
+	}()
+
+	dec := codec.NewDecoder(f, cfg.MaxKeySize, cfg.MaxValueSize)
+	enc := codec.NewEncoder(fr)
+	e := internal.Entry{}
+
+	corrupted := false
+	for !corrupted {
+		_, err = dec.Decode(&e)
+		if err == io.EOF {
+			break
+		}
+		if codec.IsCorruptedData(err) {
+			corrupted = true
+			continue
+		}
+		if err != nil {
+			return false, fmt.Errorf("unexpected error while reading datafile: %w", err)
+		}
+		if _, err := enc.Encode(e); err != nil {
+			return false, fmt.Errorf("writing to recovered datafile: %w", err)
+		}
+	}
+	if !corrupted {
+		if err := os.Remove(fr.Name()); err != nil {
+			return false, fmt.Errorf("can't remove temporal recovered datafile: %w", err)
+		}
+		return false, nil
+	}
+	if err := os.Rename(rPath, path); err != nil {
+		return false, fmt.Errorf("removing corrupted file: %s", err)
+	}
+	return true, nil
+}

v2/internal/entry.go

@@ -0,0 +1,27 @@
+package internal
+
+import (
+	"hash/crc32"
+	"time"
+)
+
+// Entry represents a key/value in the database
+type Entry struct {
+	Checksum uint32
+	Key      []byte
+	Offset   int64
+	Value    []byte
+	Expiry   *time.Time
+}
+
+// NewEntry creates a new `Entry` with the given `key` and `value`
+func NewEntry(key, value []byte, expiry *time.Time) Entry {
+	checksum := crc32.ChecksumIEEE(value)
+
+	return Entry{
+		Checksum: checksum,
+		Key:      key,
+		Value:    value,
+		Expiry:   expiry,
+	}
+}

v2/internal/index/codec_index.go

@@ -0,0 +1,134 @@
+package index
+
+import (
+	"encoding/binary"
+	"io"
+
+	"git.mills.io/prologic/bitcask/v2/internal"
+	"github.com/pkg/errors"
+	art "github.com/plar/go-adaptive-radix-tree"
+)
+
+var (
+	errTruncatedKeySize = errors.New("key size is truncated")
+	errTruncatedKeyData = errors.New("key data is truncated")
+	errTruncatedData    = errors.New("data is truncated")
+	errKeySizeTooLarge  = errors.New("key size too large")
+)
+
+const (
+	int32Size  = 4
+	int64Size  = 8
+	fileIDSize = int32Size
+	offsetSize = int64Size
+	sizeSize   = int64Size
+)
+
+func readKeyBytes(r io.Reader, maxKeySize uint32) ([]byte, error) {
+	s := make([]byte, int32Size)
+	_, err := io.ReadFull(r, s)
+	if err != nil {
+		if err == io.EOF {
+			return nil, err
+		}
+		return nil, errors.Wrap(errTruncatedKeySize, err.Error())
+	}
+	size := binary.BigEndian.Uint32(s)
+	if maxKeySize > 0 && size > uint32(maxKeySize) {
+		return nil, errKeySizeTooLarge
+	}
+
+	b := make([]byte, size)
+	_, err = io.ReadFull(r, b)
+	if err != nil {
+		return nil, errors.Wrap(errTruncatedKeyData, err.Error())
+	}
+	return b, nil
+}
+
+func writeBytes(b []byte, w io.Writer) error {
+	s := make([]byte, int32Size)
+	binary.BigEndian.PutUint32(s, uint32(len(b)))
+	_, err := w.Write(s)
+	if err != nil {
+		return err
+	}
+	_, err = w.Write(b)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func readItem(r io.Reader) (internal.Item, error) {
+	buf := make([]byte, (fileIDSize + offsetSize + sizeSize))
+	_, err := io.ReadFull(r, buf)
+	if err != nil {
+		return internal.Item{}, errors.Wrap(errTruncatedData, err.Error())
+	}
+
+	return internal.Item{
+		FileID: int(binary.BigEndian.Uint32(buf[:fileIDSize])),
+		Offset: int64(binary.BigEndian.Uint64(buf[fileIDSize:(fileIDSize + offsetSize)])),
+		Size:   int64(binary.BigEndian.Uint64(buf[(fileIDSize + offsetSize):])),
+	}, nil
+}
+
+func writeItem(item internal.Item, w io.Writer) error {
+	buf := make([]byte, (fileIDSize + offsetSize + sizeSize))
+	binary.BigEndian.PutUint32(buf[:fileIDSize], uint32(item.FileID))
+	binary.BigEndian.PutUint64(buf[fileIDSize:(fileIDSize+offsetSize)], uint64(item.Offset))
+	binary.BigEndian.PutUint64(buf[(fileIDSize+offsetSize):], uint64(item.Size))
+	_, err := w.Write(buf)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// ReadIndex reads a persisted from a io.Reader into a Tree
+func readIndex(r io.Reader, t art.Tree, maxKeySize uint32) error {
+	for {
+		key, err := readKeyBytes(r, maxKeySize)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return err
+		}
+
+		item, err := readItem(r)
+		if err != nil {
+			return err
+		}
+
+		t.Insert(key, item)
+	}
+
+	return nil
+}
+
+func writeIndex(t art.Tree, w io.Writer) (err error) {
+	t.ForEach(func(node art.Node) bool {
+		err = writeBytes(node.Key(), w)
+		if err != nil {
+			return false
+		}
+
+		item := node.Value().(internal.Item)
+		err := writeItem(item, w)
+		return err == nil
+	})
+	return
+}
+
+// IsIndexCorruption returns a boolean indicating whether the error
+// is known to report a corruption data issue
+func IsIndexCorruption(err error) bool {
+	cause := errors.Cause(err)
+	switch cause {
+	case errKeySizeTooLarge, errTruncatedData, errTruncatedKeyData, errTruncatedKeySize:
+		return true
+	}
+	return false
+}

v2/internal/index/codec_index_test.go

@@ -0,0 +1,126 @@
+package index
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/binary"
+	"testing"
+
+	"git.mills.io/prologic/bitcask/v2/internal"
+	"github.com/pkg/errors"
+	art "github.com/plar/go-adaptive-radix-tree"
+)
+
+const (
+	base64SampleTree = "AAAABGFiY2QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARhYmNlAAAAAQAAAAAAAAABAAAAAAAAAAEAAAAEYWJjZgAAAAIAAAAAAAAAAgAAAAAAAAACAAAABGFiZ2QAAAADAAAAAAAAAAMAAAAAAAAAAw=="
+)
+
+func TestWriteIndex(t *testing.T) {
+	at, expectedSerializedSize := getSampleTree()
+
+	var b bytes.Buffer
+	err := writeIndex(at, &b)
+	if err != nil {
+		t.Fatalf("writing index failed: %v", err)
+	}
+	if b.Len() != expectedSerializedSize {
+		t.Fatalf("incorrect size of serialied index: expected %d, got: %d", expectedSerializedSize, b.Len())
+	}
+	sampleTreeBytes, _ := base64.StdEncoding.DecodeString(base64SampleTree)
+	if !bytes.Equal(b.Bytes(), sampleTreeBytes) {
+		t.Fatalf("unexpected serialization of the tree")
+	}
+}
+
+func TestReadIndex(t *testing.T) {
+	sampleTreeBytes, _ := base64.StdEncoding.DecodeString(base64SampleTree)
+	b := bytes.NewBuffer(sampleTreeBytes)
+
+	at := art.New()
+	err := readIndex(b, at, 1024)
+	if err != nil {
+		t.Fatalf("error while deserializing correct sample tree: %v", err)
+	}
+
+	atsample, _ := getSampleTree()
+	if atsample.Size() != at.Size() {
+		t.Fatalf("trees aren't the same size, expected %v, got %v", atsample.Size(), at.Size())
+	}
+	atsample.ForEach(func(node art.Node) bool {
+		_, found := at.Search(node.Key())
+		if !found {
+			t.Fatalf("expected node wasn't found: %s", node.Key())
+		}
+		return true
+	})
+}
+
+func TestReadCorruptedData(t *testing.T) {
+	sampleBytes, _ := base64.StdEncoding.DecodeString(base64SampleTree)
+
+	t.Run("truncated", func(t *testing.T) {
+		table := []struct {
+			name string
+			err  error
+			data []byte
+		}{
+			{name: "key-size-first-item", err: errTruncatedKeySize, data: sampleBytes[:2]},
+			{name: "key-data-second-item", err: errTruncatedKeyData, data: sampleBytes[:6]},
+			{name: "key-size-second-item", err: errTruncatedKeySize, data: sampleBytes[:(int32Size+4+fileIDSize+offsetSize+sizeSize)+2]},
+			{name: "key-data-second-item", err: errTruncatedKeyData, data: sampleBytes[:(int32Size+4+fileIDSize+offsetSize+sizeSize)+6]},
+			{name: "data", err: errTruncatedData, data: sampleBytes[:int32Size+4+(fileIDSize+offsetSize+sizeSize-3)]},
+		}
+
+		for i := range table {
+			t.Run(table[i].name, func(t *testing.T) {
+				bf := bytes.NewBuffer(table[i].data)
+
+				if err := readIndex(bf, art.New(), 1024); !IsIndexCorruption(err) || errors.Cause(err) != table[i].err {
+					t.Fatalf("expected %v, got %v", table[i].err, err)
+				}
+			})
+		}
+	})
+
+	t.Run("overflow", func(t *testing.T) {
+		overflowKeySize := make([]byte, len(sampleBytes))
+		copy(overflowKeySize, sampleBytes)
+		binary.BigEndian.PutUint32(overflowKeySize, 1025)
+
+		overflowDataSize := make([]byte, len(sampleBytes))
+		copy(overflowDataSize, sampleBytes)
+		binary.BigEndian.PutUint32(overflowDataSize[int32Size+4+fileIDSize+offsetSize:], 1025)
+
+		table := []struct {
+			name       string
+			err        error
+			maxKeySize uint32
+			data       []byte
+		}{
+			{name: "key-data-overflow", err: errKeySizeTooLarge, maxKeySize: 1024, data: overflowKeySize},
+		}
+
+		for i := range table {
+			t.Run(table[i].name, func(t *testing.T) {
+				bf := bytes.NewBuffer(table[i].data)
+
+				if err := readIndex(bf, art.New(), table[i].maxKeySize); !IsIndexCorruption(err) || errors.Cause(err) != table[i].err {
+					t.Fatalf("expected %v, got %v", table[i].err, err)
+				}
+			})
+		}
+	})
+
+}
+
+func getSampleTree() (art.Tree, int) {
+	at := art.New()
+	keys := [][]byte{[]byte("abcd"), []byte("abce"), []byte("abcf"), []byte("abgd")}
+	expectedSerializedSize := 0
+	for i := range keys {
+		at.Insert(keys[i], internal.Item{FileID: i, Offset: int64(i), Size: int64(i)})
+		expectedSerializedSize += int32Size + len(keys[i]) + fileIDSize + offsetSize + sizeSize
+	}
+
+	return at, expectedSerializedSize
+}

v2/internal/index/index.go

@@ -0,0 +1,59 @@
+package index
+
+import (
+	"os"
+
+	"git.mills.io/prologic/bitcask/v2/internal"
+	art "github.com/plar/go-adaptive-radix-tree"
+)
+
+// Indexer is an interface for loading and saving the index (an Adaptive Radix Tree)
+type Indexer interface {
+	Load(path string, maxkeySize uint32) (art.Tree, bool, error)
+	Save(t art.Tree, path string) error
+}
+
+// NewIndexer returns an instance of the default `Indexer` implemtnation
+// which perists the index (an Adaptive Radix Tree) as a binary blob on file
+func NewIndexer() Indexer {
+	return &indexer{}
+}
+
+type indexer struct{}
+
+func (i *indexer) Load(path string, maxKeySize uint32) (art.Tree, bool, error) {
+	t := art.New()
+
+	if !internal.Exists(path) {
+		return t, false, nil
+	}
+
+	f, err := os.Open(path)
+	if err != nil {
+		return t, true, err
+	}
+	defer f.Close()
+
+	if err := readIndex(f, t, maxKeySize); err != nil {
+		return t, true, err
+	}
+	return t, true, nil
+}
+
+func (i *indexer) Save(t art.Tree, path string) error {
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	if err := writeIndex(t, f); err != nil {
+		return err
+	}
+
+	if err := f.Sync(); err != nil {
+		return err
+	}
+
+	return f.Close()
+}

v2/internal/index/ttl_index.go

@@ -0,0 +1,71 @@
+package index
+
+import (
+	"encoding/binary"
+	"io"
+	"os"
+	"time"
+
+	"git.mills.io/prologic/bitcask/v2/internal"
+	art "github.com/plar/go-adaptive-radix-tree"
+)
+
+type ttlIndexer struct{}
+
+func NewTTLIndexer() Indexer {
+	return ttlIndexer{}
+}
+
+func (i ttlIndexer) Save(t art.Tree, path string) error {
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	buf := make([]byte, int64Size)
+	for it := t.Iterator(); it.HasNext(); {
+		node, err := it.Next()
+		if err != nil {
+			return err
+		}
+		// save key
+		err = writeBytes(node.Key(), f)
+		if err != nil {
+			return err
+		}
+		// save key ttl
+		binary.BigEndian.PutUint64(buf, uint64(node.Value().(time.Time).Unix()))
+		_, err = f.Write(buf)
+		if err != nil {
+			return err
+		}
+	}
+	return f.Sync()
+}
+
+func (i ttlIndexer) Load(path string, maxKeySize uint32) (art.Tree, bool, error) {
+	t := art.New()
+	if !internal.Exists(path) {
+		return t, false, nil
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return t, true, err
+	}
+	buf := make([]byte, int64Size)
+	for {
+		key, err := readKeyBytes(f, maxKeySize)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return t, true, err
+		}
+		_, err = io.ReadFull(f, buf)
+		if err != nil {
+			return t, true, err
+		}
+		expiry := time.Unix(int64(binary.BigEndian.Uint64(buf)), 0).UTC()
+		t.Insert(key, expiry)
+	}
+	return t, true, nil
+}

v2/internal/index/ttl_index_test.go

@@ -0,0 +1,54 @@
+package index
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	art "github.com/plar/go-adaptive-radix-tree"
+	assert2 "github.com/stretchr/testify/assert"
+)
+
+func Test_TTLIndexer(t *testing.T) {
+	assert := assert2.New(t)
+	tempDir, err := ioutil.TempDir("", "bitcask")
+	assert.NoError(err)
+	defer os.RemoveAll(tempDir)
+
+	currTime := time.Date(2020, 12, 27, 0, 0, 0, 0, time.UTC)
+	trie := art.New()
+
+	t.Run("LoadEmpty", func(t *testing.T) {
+		newTrie, found, err := NewTTLIndexer().Load(filepath.Join(tempDir, "ttl_index"), 4)
+		assert.NoError(err)
+		assert.False(found)
+		assert.Equal(trie, newTrie)
+	})
+
+	t.Run("Save", func(t *testing.T) {
+		trie.Insert([]byte("key"), currTime)
+		err := NewTTLIndexer().Save(trie, filepath.Join(tempDir, "ttl_index"))
+		assert.NoError(err)
+		trie.Insert([]byte("foo"), currTime.Add(24*time.Hour))
+		err = NewTTLIndexer().Save(trie, filepath.Join(tempDir, "ttl_index"))
+		assert.NoError(err)
+		trie.Insert([]byte("key"), currTime.Add(-24*time.Hour))
+		err = NewTTLIndexer().Save(trie, filepath.Join(tempDir, "ttl_index"))
+		assert.NoError(err)
+	})
+
+	t.Run("Load", func(t *testing.T) {
+		newTrie, found, err := NewTTLIndexer().Load(filepath.Join(tempDir, "ttl_index"), 4)
+		assert.NoError(err)
+		assert.True(found)
+		assert.Equal(2, newTrie.Size())
+		value, found := newTrie.Search([]byte("key"))
+		assert.True(found)
+		assert.Equal(currTime.Add(-24*time.Hour), value)
+		value, found = newTrie.Search([]byte("foo"))
+		assert.True(found)
+		assert.Equal(currTime.Add(24*time.Hour), value)
+	})
+}

v2/internal/item.go

@@ -0,0 +1,10 @@
+package internal
+
+// Item represents the location of the value on disk. This is used by the
+// internal Adaptive Radix Tree to hold an in-memory structure mapping keys to
+// locations on disk of where the value(s) can be read from.
+type Item struct {
+	FileID int   `json:"fileid"`
+	Offset int64 `json:"offset"`
+	Size   int64 `json:"size"`
+}

v2/internal/metadata/metadata.go

@@ -0,0 +1,22 @@
+package metadata
+
+import (
+	"os"
+
+	"git.mills.io/prologic/bitcask/v2/internal"
+)
+
+type MetaData struct {
+	IndexUpToDate    bool  `json:"index_up_to_date"`
+	ReclaimableSpace int64 `json:"reclaimable_space"`
+}
+
+func (m *MetaData) Save(path string, mode os.FileMode) error {
+	return internal.SaveJsonToFile(m, path, mode)
+}
+
+func Load(path string) (*MetaData, error) {
+	var m MetaData
+	err := internal.LoadFromJsonFile(path, &m)
+	return &m, err
+}

v2/internal/mocks/datafile.go

@@ -0,0 +1,158 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+package mocks
+
+import internal "git.mills.io/prologic/bitcask/v2/internal"
+import mock "github.com/stretchr/testify/mock"
+
+// Datafile is an autogenerated mock type for the Datafile type
+type Datafile struct {
+	mock.Mock
+}
+
+// Close provides a mock function with given fields:
+func (_m *Datafile) Close() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// FileID provides a mock function with given fields:
+func (_m *Datafile) FileID() int {
+	ret := _m.Called()
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func() int); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	return r0
+}
+
+// Name provides a mock function with given fields:
+func (_m *Datafile) Name() string {
+	ret := _m.Called()
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func() string); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	return r0
+}
+
+// Read provides a mock function with given fields:
+func (_m *Datafile) Read() (internal.Entry, int64, error) {
+	ret := _m.Called()
+
+	var r0 internal.Entry
+	if rf, ok := ret.Get(0).(func() internal.Entry); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(internal.Entry)
+	}
+
+	var r1 int64
+	if rf, ok := ret.Get(1).(func() int64); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Get(1).(int64)
+	}
+
+	var r2 error
+	if rf, ok := ret.Get(2).(func() error); ok {
+		r2 = rf()
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
+}
+
+// ReadAt provides a mock function with given fields: index, size
+func (_m *Datafile) ReadAt(index int64, size int64) (internal.Entry, error) {
+	ret := _m.Called(index, size)
+
+	var r0 internal.Entry
+	if rf, ok := ret.Get(0).(func(int64, int64) internal.Entry); ok {
+		r0 = rf(index, size)
+	} else {
+		r0 = ret.Get(0).(internal.Entry)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(int64, int64) error); ok {
+		r1 = rf(index, size)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Size provides a mock function with given fields:
+func (_m *Datafile) Size() int64 {
+	ret := _m.Called()
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func() int64); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	return r0
+}
+
+// Sync provides a mock function with given fields:
+func (_m *Datafile) Sync() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Write provides a mock function with given fields: _a0
+func (_m *Datafile) Write(_a0 internal.Entry) (int64, int64, error) {
+	ret := _m.Called(_a0)
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func(internal.Entry) int64); ok {
+		r0 = rf(_a0)
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	var r1 int64
+	if rf, ok := ret.Get(1).(func(internal.Entry) int64); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Get(1).(int64)
+	}
+
+	var r2 error
+	if rf, ok := ret.Get(2).(func(internal.Entry) error); ok {
+		r2 = rf(_a0)
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
+}

v2/internal/mocks/indexer.go

@@ -0,0 +1,56 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+package mocks
+
+import art "github.com/plar/go-adaptive-radix-tree"
+
+import mock "github.com/stretchr/testify/mock"
+
+// Indexer is an autogenerated mock type for the Indexer type
+type Indexer struct {
+	mock.Mock
+}
+
+// Load provides a mock function with given fields: path, maxkeySize
+func (_m *Indexer) Load(path string, maxkeySize uint32) (art.Tree, bool, error) {
+	ret := _m.Called(path, maxkeySize)
+
+	var r0 art.Tree
+	if rf, ok := ret.Get(0).(func(string, uint32) art.Tree); ok {
+		r0 = rf(path, maxkeySize)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(art.Tree)
+		}
+	}
+
+	var r1 bool
+	if rf, ok := ret.Get(1).(func(string, uint32) bool); ok {
+		r1 = rf(path, maxkeySize)
+	} else {
+		r1 = ret.Get(1).(bool)
+	}
+
+	var r2 error
+	if rf, ok := ret.Get(2).(func(string, uint32) error); ok {
+		r2 = rf(path, maxkeySize)
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
+}
+
+// Save provides a mock function with given fields: t, path
+func (_m *Indexer) Save(t art.Tree, path string) error {
+	ret := _m.Called(t, path)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(art.Tree, string) error); ok {
+		r0 = rf(t, path)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}

v2/internal/utils.go

@@ -0,0 +1,112 @@
+package internal
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// Exists returns `true` if the given `path` on the current file system exists
+func Exists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+// DirSize returns the space occupied by the given `path` on disk on the current
+// file system.
+func DirSize(path string) (int64, error) {
+	var size int64
+	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			size += info.Size()
+		}
+		return err
+	})
+	return size, err
+}
+
+// GetDatafiles returns a list of all data files stored in the database path
+// given by `path`. All datafiles are identified by the the glob `*.data` and
+// the basename is represented by a monotonic increasing integer.
+// The returned files are *sorted* in increasing order.
+func GetDatafiles(path string) ([]string, error) {
+	fns, err := filepath.Glob(fmt.Sprintf("%s/*.data", path))
+	if err != nil {
+		return nil, err
+	}
+	sort.Strings(fns)
+	return fns, nil
+}
+
+// ParseIds will parse a list of datafiles as returned by `GetDatafiles` and
+// extract the id part and return a slice of ints.
+func ParseIds(fns []string) ([]int, error) {
+	var ids []int
+	for _, fn := range fns {
+		fn = filepath.Base(fn)
+		ext := filepath.Ext(fn)
+		if ext != ".data" {
+			continue
+		}
+		id, err := strconv.ParseInt(strings.TrimSuffix(fn, ext), 10, 32)
+		if err != nil {
+			return nil, err
+		}
+		ids = append(ids, int(id))
+	}
+	sort.Ints(ids)
+	return ids, nil
+}
+
+// Copy copies source contents to destination
+func Copy(src, dst string, exclude []string) error {
+	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+		relPath := strings.Replace(path, src, "", 1)
+		if relPath == "" {
+			return nil
+		}
+		for _, e := range exclude {
+			matched, err := filepath.Match(e, info.Name())
+			if err != nil {
+				return err
+			}
+			if matched {
+				return nil
+			}
+		}
+		if info.IsDir() {
+			return os.Mkdir(filepath.Join(dst, relPath), info.Mode())
+		}
+		var data, err1 = ioutil.ReadFile(filepath.Join(src, relPath))
+		if err1 != nil {
+			return err1
+		}
+		return ioutil.WriteFile(filepath.Join(dst, relPath), data, info.Mode())
+	})
+}
+
+// SaveJsonToFile converts v into json and store in file identified by path
+func SaveJsonToFile(v interface{}, path string, mode os.FileMode) error {
+	b, err := json.Marshal(v)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(path, b, mode)
+}
+
+// LoadFromJsonFile reads file located at `path` and put its content in json format in v
+func LoadFromJsonFile(path string, v interface{}) error {
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(b, v)
+}

v2/internal/utils_test.go

@@ -0,0 +1,108 @@
+package internal
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_Copy(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("CopyDir", func(t *testing.T) {
+		tempsrc, err := ioutil.TempDir("", "test")
+		assert.NoError(err)
+		defer os.RemoveAll(tempsrc)
+		var f *os.File
+
+		tempdir, err := ioutil.TempDir(tempsrc, "")
+		assert.NoError(err)
+
+		f, err = os.OpenFile(filepath.Join(tempsrc, "file1"), os.O_WRONLY|os.O_CREATE, 0755)
+		assert.NoError(err)
+		n, err := f.WriteString("test123")
+		assert.Equal(7, n)
+		assert.NoError(err)
+		f.Close()
+
+		f, err = os.OpenFile(filepath.Join(tempsrc, "file2"), os.O_WRONLY|os.O_CREATE, 0755)
+		assert.NoError(err)
+		n, err = f.WriteString("test1234")
+		assert.Equal(8, n)
+		assert.NoError(err)
+		f.Close()
+
+		f, err = os.OpenFile(filepath.Join(tempsrc, "file3"), os.O_WRONLY|os.O_CREATE, 0755)
+		assert.NoError(err)
+		f.Close()
+
+		tempdst, err := ioutil.TempDir("", "backup")
+		assert.NoError(err)
+		defer os.RemoveAll(tempdst)
+		err = Copy(tempsrc, tempdst, []string{"file3"})
+		assert.NoError(err)
+		buf := make([]byte, 10)
+
+		exists := Exists(filepath.Join(tempdst, filepath.Base(tempdir)))
+		assert.Equal(true, exists)
+
+		f, err = os.Open(filepath.Join(tempdst, "file1"))
+		assert.NoError(err)
+		n, err = f.Read(buf[:7])
+		assert.NoError(err)
+		assert.Equal(7, n)
+		assert.Equal([]byte("test123"), buf[:7])
+		_, err = f.Read(buf)
+		assert.Equal(io.EOF, err)
+		f.Close()
+
+		f, err = os.Open(filepath.Join(tempdst, "file2"))
+		assert.NoError(err)
+		n, err = f.Read(buf[:8])
+		assert.NoError(err)
+		assert.Equal(8, n)
+		assert.Equal([]byte("test1234"), buf[:8])
+		_, err = f.Read(buf)
+		assert.Equal(io.EOF, err)
+		f.Close()
+
+		exists = Exists(filepath.Join(tempdst, "file3"))
+		assert.Equal(false, exists)
+	})
+}
+
+func Test_SaveAndLoad(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("save and load", func(t *testing.T) {
+		tempdir, err := ioutil.TempDir("", "bitcask")
+		assert.NoError(err)
+		defer os.RemoveAll(tempdir)
+		type test struct {
+			Value bool `json:"value"`
+		}
+		m := test{Value: true}
+		err = SaveJsonToFile(&m, filepath.Join(tempdir, "meta.json"), 0755)
+		assert.NoError(err)
+		m1 := test{}
+		err = LoadFromJsonFile(filepath.Join(tempdir, "meta.json"), &m1)
+		assert.NoError(err)
+		assert.Equal(m, m1)
+	})
+
+	t.Run("save and load error", func(t *testing.T) {
+		tempdir, err := ioutil.TempDir("", "bitcask")
+		assert.NoError(err)
+		defer os.RemoveAll(tempdir)
+		type test struct {
+			Value bool `json:"value"`
+		}
+		err = SaveJsonToFile(make(chan int), filepath.Join(tempdir, "meta.json"), 0755)
+		assert.Error(err)
+		m1 := test{}
+		err = LoadFromJsonFile(filepath.Join(tempdir, "meta.json"), &m1)
+		assert.Error(err)
+	})
+}

v2/internal/version.go

@@ -0,0 +1,18 @@
+package internal
+
+import (
+	"fmt"
+)
+
+var (
+	// Version release version
+	Version = "0.0.1"
+
+	// Commit will be overwritten automatically by the build system
+	Commit = "HEAD"
+)
+
+// FullVersion returns the full version and commit hash
+func FullVersion() string {
+	return fmt.Sprintf("%s@%s", Version, Commit)
+}

v2/internal/version_test.go

@@ -0,0 +1,15 @@
+package internal
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFullVersion(t *testing.T) {
+	assert := assert.New(t)
+
+	expected := fmt.Sprintf("%s@%s", Version, Commit)
+	assert.Equal(expected, FullVersion())
+}