Improve Get/Put performance with optional mempooling (#36)

* avoid unnecessary use of encoder/decoder to decrease memory allocations

* add an optional configurable mempool to avoid extra allocs

* add doc.go with examples

CONTRIBUTING.md

@@ -0,0 +1,8 @@
+# Contributing
+
+No preference. If you know hot to use Github and contributed to open source projects before then:
+
+* File an issue
+* Submit a pull request
+* File an issue + Submit a pull request
+* Use this project somewhere :)

Dockerfile

@@ -0,0 +1,14 @@
+# Build
+FROM prologic/go-builder:latest AS build
+
+# Runtime
+FROM alpine
+
+COPY --from=build /src/bitcaskd /bitcaskd
+
+EXPOSE 6379/tcp
+
+VOLUME /data
+
+ENTRYPOINT ["/bitcaskd"]
+CMD ["/data"]

README.md

@@ -4,9 +4,11 @@
 [![CodeCov](https://codecov.io/gh/prologic/bitcask/branch/master/graph/badge.svg)](https://codecov.io/gh/prologic/bitcask)
 [![Go Report Card](https://goreportcard.com/badge/prologic/bitcask)](https://goreportcard.com/report/prologic/bitcask)
 [![GoDoc](https://godoc.org/github.com/prologic/bitcask?status.svg)](https://godoc.org/github.com/prologic/bitcask) 
-[![Sourcegraph](https://sourcegraph.com/github.com/prologic/bitcask/-/badge.svg)](https://sourcegraph.com/github.com/prologic/bitcask?badge)
+[![GitHub license](https://img.shields.io/github/license/prologic/bitcask.svg)](https://github.com/prologic/bitcask)
 
-A high performance Key/Value store written in [Go](https://golang.org) with a predictable read/write performance and high throughput. Uses a [Bitcask](https://en.wikipedia.org/wiki/Bitcask) on-disk layout (LSM+WAL) similar to [Riak](https://riak.com/).
+A high performance Key/Value store written in [Go](https://golang.org) with a predictable read/write performance and high throughput. Uses a [Bitcask](https://en.wikipedia.org/wiki/Bitcask) on-disk layout (LSM+WAL) similar to [Riak](https://riak.com/)
+
+For a more feature-complete Redis-compatible server, distributed key/value store have a look at [Bitraft](https://github.com/prologic/bitraft) which uses this library as its backend. Use [Bitcask](https://github.com/prologic/bitcask) as a starting point or if you want to embed in your application, use [Bitraft](https://github.com/prologic/bitraft) if you need a complete server/client solution with high availability with a Redis-compatible API.
 
 ## Features
 
@@ -14,12 +16,35 @@ A high performance Key/Value store written in [Go](https://golang.org) with a pr
 * Builtin CLI (`bitcask`)
 * Builtin Redis-compatible server (`bitcaskd`)
 * Predictable read/write performance
-* Low latecny
+* Low latency
-* High throughput (See: [Performance](README.md#Performance)
+* High throughput (See: [Performance](README.md#Performance) )
+
+## Development
+
+1. Get the source
+
+```#!sh
+$ git clone https://github.com/prologic/bitcask.git
+```
+
+2. Install required tools
+
+This library uses [Protobuf](https://github.com/protocolbuffers/protobuf) to serialize data on disk. Please follow the
+instructions for installing `protobuf` on your system. You will also need the
+following Go libraries/tools to generate Go code from Protobuf defs:
+- [protoc-gen-go](https://github.com/golang/protobuf)
+
+3. Build the project
+
+```#!sh
+$ make
+```
+
+This will invoke `go generate` and `go build`.
 
 ## Install
 
-```#!bash
+```#!sh
 $ go get github.com/prologic/bitcask
 ```
 
@@ -27,7 +52,7 @@ $ go get github.com/prologic/bitcask
 
 Install the package into your project:
 
-```#!bash
+```#!sh
 $ go get github.com/prologic/bitcask
 ```
 
@@ -39,8 +64,8 @@ import "github.com/prologic/bitcask"
 func main() {
     db, _ := bitcask.Open("/tmp/db")
     defer db.Close()
-    db.Set("Hello", []byte("World"))
+    db.Put("Hello", []byte("World"))
-    val, _ := db.Get("hello")
+    val, _ := db.Get("Hello")
 }
 ```
 
@@ -49,7 +74,7 @@ documentation and other examples.
 
 ## Usage (tool)
 
-```#!bash
+```#!sh
 $ bitcask -p /tmp/db set Hello World
 $ bitcask -p /tmp/db get Hello
 World
@@ -59,14 +84,14 @@ World
 
 There is also a builtin very  simple Redis-compatible server called `bitcaskd`:
 
-```#!bash
+```#!sh
 $ ./bitcaskd ./tmp
 INFO[0000] starting bitcaskd v0.0.7@146f777              bind=":6379" path=./tmp
 ```
 
 Example session:
 
-```
+```#!sh
 $ telnet localhost 6379
 Trying ::1...
 Connected to localhost.
@@ -87,39 +112,48 @@ QUIT
 Connection closed by foreign host.
 ```
 
+## Docker
+
+You can also use the [Bitcask Docker Image](https://cloud.docker.com/u/prologic/repository/docker/prologic/bitcask):
+
+```#!sh
+$ docker pull prologic/bitcask
+$ docker run -d -p 6379:6379 prologic/bitcask
+```
+
 ## Performance
 
 Benchmarks run on a 11" Macbook with a 1.4Ghz Intel Core i7:
 
-```
+```#!sh
 $ make bench
 ...
-BenchmarkGet/128B-4         	  300000	      3737 ns/op	    1632 B/op	      16 allocs/op
+BenchmarkGet/128B-4         	  300000	      4071 ns/op	  31.43 MB/s	     608 B/op	       7 allocs/op
-BenchmarkGet/256B-4         	  300000	      4183 ns/op	    2016 B/op	      16 allocs/op
+BenchmarkGet/256B-4         	  300000	      4700 ns/op	  54.46 MB/s	     992 B/op	       7 allocs/op
-BenchmarkGet/512B-4         	  300000	      4295 ns/op	    2848 B/op	      16 allocs/op
+BenchmarkGet/512B-4         	  300000	      4915 ns/op	 104.17 MB/s	    1824 B/op	       7 allocs/op
-BenchmarkGet/1K-4           	  300000	      4455 ns/op	    4512 B/op	      16 allocs/op
+BenchmarkGet/1K-4           	  200000	      5064 ns/op	 202.20 MB/s	    3488 B/op	       7 allocs/op
-BenchmarkGet/2K-4           	  300000	      5536 ns/op	    7841 B/op	      16 allocs/op
+BenchmarkGet/2K-4           	  200000	      6276 ns/op	 326.31 MB/s	    6816 B/op	       7 allocs/op
-BenchmarkGet/4K-4           	  200000	      7101 ns/op	   15010 B/op	      16 allocs/op
+BenchmarkGet/4K-4           	  200000	      8960 ns/op	 457.11 MB/s	   13984 B/op	       7 allocs/op
-BenchmarkGet/8K-4           	  200000	     10664 ns/op	   28325 B/op	      16 allocs/op
+BenchmarkGet/8K-4           	  100000	     12465 ns/op	 657.16 MB/s	   27296 B/op	       7 allocs/op
-BenchmarkGet/16K-4          	  100000	     18173 ns/op	   54442 B/op	      16 allocs/op
+BenchmarkGet/16K-4          	  100000	     19233 ns/op	 851.84 MB/s	   53408 B/op	       7 allocs/op
-BenchmarkGet/32K-4          	   50000	     33081 ns/op	  115893 B/op	      16 allocs/op
+BenchmarkGet/32K-4          	   50000	     33106 ns/op	 989.77 MB/s	  114848 B/op	       7 allocs/op
 
-BenchmarkPut/128B-4         	  200000	      7967 ns/op	     409 B/op	       6 allocs/op
+BenchmarkPut/128B-4         	  100000	     13659 ns/op	   9.37 MB/s	     409 B/op	       6 allocs/op
-BenchmarkPut/256B-4         	  200000	      8563 ns/op	     538 B/op	       6 allocs/op
+BenchmarkPut/256B-4         	  100000	     14854 ns/op	  17.23 MB/s	     539 B/op	       6 allocs/op
-BenchmarkPut/512B-4         	  200000	      9678 ns/op	     829 B/op	       6 allocs/op
+BenchmarkPut/512B-4         	  100000	     20823 ns/op	  24.59 MB/s	     829 B/op	       6 allocs/op
-BenchmarkPut/1K-4           	  200000	     12786 ns/op	    1410 B/op	       6 allocs/op
+BenchmarkPut/1K-4           	   50000	     28086 ns/op	  36.46 MB/s	    1411 B/op	       6 allocs/op
-BenchmarkPut/2K-4           	  100000	     18582 ns/op	    2572 B/op	       6 allocs/op
+BenchmarkPut/2K-4           	   30000	     40797 ns/op	  50.20 MB/s	    2574 B/op	       6 allocs/op
-BenchmarkPut/4K-4           	   50000	     35335 ns/op	    5151 B/op	       6 allocs/op
+BenchmarkPut/4K-4           	   20000	     75518 ns/op	  54.24 MB/s	    5155 B/op	       6 allocs/op
-BenchmarkPut/8K-4           	   30000	     56047 ns/op	    9797 B/op	       6 allocs/op
+BenchmarkPut/8K-4           	   10000	    122544 ns/op	  66.85 MB/s	    9811 B/op	       6 allocs/op
-BenchmarkPut/16K-4          	   20000	     86137 ns/op	   18834 B/op	       6 allocs/op
+BenchmarkPut/16K-4          	   10000	    201167 ns/op	  81.44 MB/s	   18851 B/op	       6 allocs/op
-BenchmarkPut/32K-4          	   10000	    140162 ns/op	   41517 B/op	       6 allocs/op
+BenchmarkPut/32K-4          	    5000	    350850 ns/op	  93.40 MB/s	   41565 B/op	       7 allocs/op
 
-BenchmarkScan-4             	 1000000	      1885 ns/op	     493 B/op	      25 allocs/op
+BenchmarkScan-4             	 1000000	      1867 ns/op	     493 B/op	      25 allocs/op
 ```
 
 For 128B values:
 
-* ~270,000 reads/sec
+* ~400,000 reads/sec
 * ~130,000 writes/sec
 
 The full benchmark above shows linear performance as you increase key/value sizes.

bitcask.go

@@ -6,8 +6,8 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
-	"strings"
 	"sync"
 
 	"github.com/gofrs/flock"
@@ -28,13 +28,17 @@ var (
 	// maximum allowed value size (configured with WithMaxValueSize).
 	ErrValueTooLarge = errors.New("error: value too large")
 
-	// ErrChecksumFailed is the error returned if a key/valie retrieved does
+	// ErrChecksumFailed is the error returned if a key/value retrieved does
 	// not match its CRC checksum
 	ErrChecksumFailed = errors.New("error: checksum failed")
 
 	// ErrDatabaseLocked is the error returned if the database is locked
 	// (typically opened by another process)
 	ErrDatabaseLocked = errors.New("error: database locked")
+
+	// ErrCreatingMemPool is the error returned when trying to configurate
+	// the mempool fails
+	ErrCreatingMemPool = errors.New("error: creating the mempool failed")
 )
 
 // Bitcask is a struct that represents a on-disk LSM and WAL data structure
@@ -46,15 +50,40 @@ type Bitcask struct {
 	*flock.Flock
 
 	config    *config
+	options   []Option
 	path      string
 	curr      *internal.Datafile
 	keydir    *internal.Keydir
-	datafiles []*internal.Datafile
+	datafiles map[int]*internal.Datafile
 	trie      *trie.Trie
 }
 
+// Stats is a struct returned by Stats() on an open Bitcask instance
+type Stats struct {
+	Datafiles int
+	Keys      int
+	Size      int64
+}
+
+// Stats returns statistics about the database including the number of
+// data files, keys and overall size on disk of the data
+func (b *Bitcask) Stats() (stats Stats, err error) {
+	var size int64
+
+	size, err = internal.DirSize(b.path)
+	if err != nil {
+		return
+	}
+
+	stats.Datafiles = len(b.datafiles)
+	stats.Keys = b.keydir.Len()
+	stats.Size = size
+
+	return
+}
+
 // Close closes the database and removes the lock. It is important to call
-// Close() as this is the only wat to cleanup the lock held by the open
+// Close() as this is the only way to cleanup the lock held by the open
 // database.
 func (b *Bitcask) Close() error {
 	defer func() {
@@ -62,9 +91,16 @@ func (b *Bitcask) Close() error {
 		os.Remove(b.Flock.Path())
 	}()
 
-	for _, df := range b.datafiles {
+	if err := b.keydir.Save(path.Join(b.path, "index")); err != nil {
-		df.Close()
+		return err
 	}
+
+	for _, df := range b.datafiles {
+		if err := df.Close(); err != nil {
+			return err
+		}
+	}
+
 	return b.curr.Close()
 }
 
@@ -74,7 +110,7 @@ func (b *Bitcask) Sync() error {
 }
 
 // Get retrieves the value of the given key. If the key is not found or an/I/O
-// error occurs a null byte slice is returend along with the error.
+// error occurs a null byte slice is returned along with the error.
 func (b *Bitcask) Get(key string) ([]byte, error) {
 	var df *internal.Datafile
 
@@ -142,7 +178,7 @@ func (b *Bitcask) Delete(key string) error {
 	return nil
 }
 
-// Scan performa a prefix scan of keys matching the given prefix and calling
+// Scan performs a prefix scan of keys matching the given prefix and calling
 // the function `f` with the keys found. If the function returns an error
 // no further keys are processed and the first error returned.
 func (b *Bitcask) Scan(prefix string, f func(key string) error) error {
@@ -188,14 +224,16 @@ func (b *Bitcask) put(key string, value []byte) (int64, int64, error) {
 			return -1, 0, err
 		}
 
-		df, err := internal.NewDatafile(b.path, b.curr.FileID(), true)
+		id := b.curr.FileID()
+
+		df, err := internal.NewDatafile(b.path, id, true)
 		if err != nil {
 			return -1, 0, err
 		}
 
-		b.datafiles = append(b.datafiles, df)
+		b.datafiles[id] = df
 
-		id := b.curr.FileID() + 1
+		id = b.curr.FileID() + 1
 		curr, err := internal.NewDatafile(b.path, id, false)
 		if err != nil {
 			return -1, 0, err
@@ -207,11 +245,11 @@ func (b *Bitcask) put(key string, value []byte) (int64, int64, error) {
 	return b.curr.Write(e)
 }
 
-// Merge merges all datafiles in the database creating hint files for faster
+func (b *Bitcask) reopen() error {
-// startup. Old keys are squashed and deleted keys removes. Call this function
+	b.mu.Lock()
-// periodically to reclaim disk space.
+	defer b.mu.Unlock()
-func Merge(path string, force bool) error {
+
-	fns, err := internal.GetDatafiles(path)
+	fns, err := internal.GetDatafiles(b.path)
 	if err != nil {
 		return err
 	}
@@ -221,161 +259,36 @@ func Merge(path string, force bool) error {
 		return err
 	}
 
-	// Do not merge if we only have 1 Datafile
+	datafiles := make(map[int]*internal.Datafile, len(ids))
-	if len(ids) <= 1 {
-		return nil
-	}
 
-	// Don't merge the Active Datafile (the last one)
+	for _, id := range ids {
-	fns = fns[:len(fns)-1]
+		df, err := internal.NewDatafile(b.path, id, true)
-	ids = ids[:len(ids)-1]
-
-	temp, err := ioutil.TempDir("", "bitcask")
-	if err != nil {
-		return err
-	}
-
-	for i, fn := range fns {
-		// Don't merge Datafiles whose .hint files we've already generated
-		// (they are already merged); unless we set the force flag to true
-		// (forcing a re-merge).
-		if filepath.Ext(fn) == ".hint" && !force {
-			// Already merged
-			continue
-		}
-
-		id := ids[i]
-
-		keydir := internal.NewKeydir()
-
-		df, err := internal.NewDatafile(path, id, true)
-		if err != nil {
-			return err
-		}
-		defer df.Close()
-
-		for {
-			e, n, err := df.Read()
-			if err != nil {
-				if err == io.EOF {
-					break
-				}
-				return err
-			}
-
-			// Tombstone value  (deleted key)
-			if len(e.Value) == 0 {
-				keydir.Delete(e.Key)
-				continue
-			}
-
-			keydir.Add(e.Key, ids[i], e.Offset, n)
-		}
-
-		tempdf, err := internal.NewDatafile(temp, id, false)
-		if err != nil {
-			return err
-		}
-		defer tempdf.Close()
-
-		for key := range keydir.Keys() {
-			item, _ := keydir.Get(key)
-			e, err := df.ReadAt(item.Offset, item.Size)
-			if err != nil {
-				return err
-			}
-
-			_, _, err = tempdf.Write(e)
-			if err != nil {
-				return err
-			}
-		}
-
-		err = tempdf.Close()
-		if err != nil {
-			return err
-		}
-
-		err = df.Close()
-		if err != nil {
-			return err
-		}
-
-		err = os.Rename(tempdf.Name(), df.Name())
-		if err != nil {
-			return err
-		}
-
-		hint := strings.TrimSuffix(df.Name(), ".data") + ".hint"
-		err = keydir.Save(hint)
 		if err != nil {
 			return err
 		}
+		datafiles[id] = df
 	}
 
-	return nil
-}
-
-// Open opens the database at the given path with optional options.
-// Options can be provided with the `WithXXX` functions that provide
-// configuration options as functions.
-func Open(path string, options ...Option) (*Bitcask, error) {
-	if err := os.MkdirAll(path, 0755); err != nil {
-		return nil, err
-	}
-
-	err := Merge(path, false)
-	if err != nil {
-		return nil, err
-	}
-
-	fns, err := internal.GetDatafiles(path)
-	if err != nil {
-		return nil, err
-	}
-
-	ids, err := internal.ParseIds(fns)
-	if err != nil {
-		return nil, err
-	}
-
-	var datafiles []*internal.Datafile
-
 	keydir := internal.NewKeydir()
 	trie := trie.New()
 
-	for i, fn := range fns {
+	if internal.Exists(path.Join(b.path, "index")) {
-		df, err := internal.NewDatafile(path, ids[i], true)
+		if err := keydir.Load(path.Join(b.path, "index")); err != nil {
-		if err != nil {
+			return err
-			return nil, err
 		}
-		datafiles = append(datafiles, df)
+		for key := range keydir.Keys() {
-
+			item, _ := keydir.Get(key)
-		if filepath.Ext(fn) == ".hint" {
+			trie.Add(key, item)
-			f, err := os.Open(filepath.Join(path, fn))
+		}
-			if err != nil {
+	} else {
-				return nil, err
+		for i, df := range datafiles {
-			}
-			defer f.Close()
-
-			hint, err := internal.NewKeydirFromBytes(f)
-			if err != nil {
-				return nil, err
-			}
-
-			for key := range hint.Keys() {
-				item, _ := hint.Get(key)
-				_ = keydir.Add(key, item.FileID, item.Offset, item.Size)
-				trie.Add(key, item)
-			}
-		} else {
 			for {
 				e, n, err := df.Read()
 				if err != nil {
 					if err == io.EOF {
 						break
 					}
-					return nil, err
+					return err
 				}
 
 				// Tombstone value  (deleted key)
@@ -395,28 +308,124 @@ func Open(path string, options ...Option) (*Bitcask, error) {
 		id = ids[(len(ids) - 1)]
 	}
 
-	curr, err := internal.NewDatafile(path, id, false)
+	curr, err := internal.NewDatafile(b.path, id, false)
 	if err != nil {
+		return err
+	}
+
+	b.curr = curr
+	b.datafiles = datafiles
+
+	b.keydir = keydir
+
+	b.trie = trie
+
+	return nil
+}
+
+// Merge merges all datafiles in the database. Old keys are squashed
+// and deleted keys removes. Duplicate key/value pairs are also removed.
+// Call this function periodically to reclaim disk space.
+func (b *Bitcask) Merge() error {
+	// Temporary merged database path
+	temp, err := ioutil.TempDir(b.path, "merge")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(temp)
+
+	// Create a merged database
+	mdb, err := Open(temp, b.options...)
+	if err != nil {
+		return err
+	}
+
+	// Rewrite all key/value pairs into merged database
+	// Doing this automatically strips deleted keys and
+	// old key/value pairs
+	err = b.Fold(func(key string) error {
+		value, err := b.Get(key)
+		if err != nil {
+			return err
+		}
+
+		if err := mdb.Put(key, value); err != nil {
+			return err
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = mdb.Close()
+	if err != nil {
+		return err
+	}
+
+	// Close the database
+	err = b.Close()
+	if err != nil {
+		return err
+	}
+
+	// Remove all data files
+	files, err := ioutil.ReadDir(b.path)
+	if err != nil {
+		return err
+	}
+	for _, file := range files {
+		if !file.IsDir() {
+			err := os.RemoveAll(path.Join([]string{b.path, file.Name()}...))
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// Rename all merged data files
+	files, err = ioutil.ReadDir(mdb.path)
+	if err != nil {
+		return err
+	}
+	for _, file := range files {
+		err := os.Rename(
+			path.Join([]string{mdb.path, file.Name()}...),
+			path.Join([]string{b.path, file.Name()}...),
+		)
+		if err != nil {
+			return err
+		}
+	}
+
+	// And finally reopen the database
+	return b.reopen()
+}
+
+// Open opens the database at the given path with optional options.
+// Options can be provided with the `WithXXX` functions that provide
+// configuration options as functions.
+func Open(path string, options ...Option) (*Bitcask, error) {
+	if err := os.MkdirAll(path, 0755); err != nil {
 		return nil, err
 	}
 
 	bitcask := &Bitcask{
-		Flock:     flock.New(filepath.Join(path, "lock")),
+		Flock:   flock.New(filepath.Join(path, "lock")),
-		config:    newDefaultConfig(),
+		config:  newDefaultConfig(),
-		path:      path,
+		options: options,
-		curr:      curr,
+		path:    path,
-		keydir:    keydir,
-		datafiles: datafiles,
-		trie:      trie,
 	}
 
 	for _, opt := range options {
-		err = opt(bitcask.config)
+		if err := opt(bitcask.config); err != nil {
-		if err != nil {
 			return nil, err
 		}
 	}
 
+	internal.ConfigureMemPool(bitcask.config.maxConcurrency)
+
 	locked, err := bitcask.Flock.TryLock()
 	if err != nil {
 		return nil, err
@@ -426,5 +435,22 @@ func Open(path string, options ...Option) (*Bitcask, error) {
 		return nil, ErrDatabaseLocked
 	}
 
+	if err := bitcask.reopen(); err != nil {
+		return nil, err
+	}
+
 	return bitcask, nil
 }
+
+// Merge calls Bitcask.Merge()
+// XXX: Deprecated; Please use the `.Merge()` method
+// XXX: This is only kept here for backwards compatibility
+//      it will be removed in future releases at some point
+func Merge(path string, force bool) error {
+	db, err := Open(path)
+	if err != nil {
+		return err
+	}
+
+	return db.Merge()
+}

bitcask_test.go

@@ -3,6 +3,7 @@ package bitcask
 import (
 	"fmt"
 	"io/ioutil"
+	"os"
 	"reflect"
 	"sort"
 	"strings"
@@ -210,38 +211,39 @@ func TestMaxValueSize(t *testing.T) {
 	})
 }
 
-func TestOpenMerge(t *testing.T) {
+func TestStats(t *testing.T) {
+	var (
+		db  *Bitcask
+		err error
+	)
+
 	assert := assert.New(t)
 
 	testdir, err := ioutil.TempDir("", "bitcask")
 	assert.NoError(err)
 
 	t.Run("Setup", func(t *testing.T) {
-		var (
-			db  *Bitcask
-			err error
-		)
-
 		t.Run("Open", func(t *testing.T) {
-			db, err = Open(testdir, WithMaxDatafileSize(32))
+			db, err = Open(testdir)
 			assert.NoError(err)
 		})
 
 		t.Run("Put", func(t *testing.T) {
-			for i := 0; i < 1024; i++ {
+			err := db.Put("foo", []byte("bar"))
-				err = db.Put(string(i), []byte(strings.Repeat(" ", 1024)))
+			assert.NoError(err)
-				assert.NoError(err)
-			}
 		})
 
 		t.Run("Get", func(t *testing.T) {
-			for i := 0; i < 32; i++ {
+			val, err := db.Get("foo")
-				err = db.Put(string(i), []byte(strings.Repeat(" ", 1024)))
+			assert.NoError(err)
-				assert.NoError(err)
+			assert.Equal([]byte("bar"), val)
-				val, err := db.Get(string(i))
+		})
-				assert.NoError(err)
+
-				assert.Equal([]byte(strings.Repeat(" ", 1024)), val)
+		t.Run("Stats", func(t *testing.T) {
-			}
+			stats, err := db.Stats()
+			assert.NoError(err)
+			assert.Equal(stats.Datafiles, 0)
+			assert.Equal(stats.Keys, 1)
 		})
 
 		t.Run("Sync", func(t *testing.T) {
@@ -254,34 +256,9 @@ func TestOpenMerge(t *testing.T) {
 			assert.NoError(err)
 		})
 	})
-
-	t.Run("Merge", func(t *testing.T) {
-		var (
-			db  *Bitcask
-			err error
-		)
-
-		t.Run("Open", func(t *testing.T) {
-			db, err = Open(testdir)
-			assert.NoError(err)
-		})
-
-		t.Run("Get", func(t *testing.T) {
-			for i := 0; i < 32; i++ {
-				val, err := db.Get(string(i))
-				assert.NoError(err)
-				assert.Equal([]byte(strings.Repeat(" ", 1024)), val)
-			}
-		})
-
-		t.Run("Close", func(t *testing.T) {
-			err = db.Close()
-			assert.NoError(err)
-		})
-	})
 }
 
-func TestMergeOpen(t *testing.T) {
+func TestMerge(t *testing.T) {
 	var (
 		db  *Bitcask
 		err error
@@ -299,22 +276,40 @@ func TestMergeOpen(t *testing.T) {
 		})
 
 		t.Run("Put", func(t *testing.T) {
-			for i := 0; i < 1024; i++ {
+			err := db.Put("foo", []byte("bar"))
-				err = db.Put(string(i), []byte(strings.Repeat(" ", 1024)))
+			assert.NoError(err)
+		})
+
+		s1, err := db.Stats()
+		assert.NoError(err)
+		assert.Equal(0, s1.Datafiles)
+		assert.Equal(1, s1.Keys)
+
+		t.Run("Put", func(t *testing.T) {
+			for i := 0; i < 10; i++ {
+				err := db.Put("foo", []byte("bar"))
 				assert.NoError(err)
 			}
 		})
 
-		t.Run("Get", func(t *testing.T) {
+		s2, err := db.Stats()
-			for i := 0; i < 32; i++ {
+		assert.NoError(err)
-				err = db.Put(string(i), []byte(strings.Repeat(" ", 1024)))
+		assert.Equal(5, s2.Datafiles)
-				assert.NoError(err)
+		assert.Equal(1, s2.Keys)
-				val, err := db.Get(string(i))
+		assert.True(s2.Size > s1.Size)
-				assert.NoError(err)
+
-				assert.Equal([]byte(strings.Repeat(" ", 1024)), val)
+		t.Run("Merge", func(t *testing.T) {
-			}
+			err := db.Merge()
+			assert.NoError(err)
 		})
 
+		s3, err := db.Stats()
+		assert.NoError(err)
+		assert.Equal(1, s3.Datafiles)
+		assert.Equal(1, s3.Keys)
+		assert.True(s3.Size > s1.Size)
+		assert.True(s3.Size < s2.Size)
+
 		t.Run("Sync", func(t *testing.T) {
 			err = db.Sync()
 			assert.NoError(err)
@@ -325,31 +320,6 @@ func TestMergeOpen(t *testing.T) {
 			assert.NoError(err)
 		})
 	})
-
-	t.Run("Merge", func(t *testing.T) {
-		t.Run("Merge", func(t *testing.T) {
-			err = Merge(testdir, true)
-			assert.NoError(err)
-		})
-
-		t.Run("Open", func(t *testing.T) {
-			db, err = Open(testdir)
-			assert.NoError(err)
-		})
-
-		t.Run("Get", func(t *testing.T) {
-			for i := 0; i < 32; i++ {
-				val, err := db.Get(string(i))
-				assert.NoError(err)
-				assert.Equal([]byte(strings.Repeat(" ", 1024)), val)
-			}
-		})
-
-		t.Run("Close", func(t *testing.T) {
-			err = db.Close()
-			assert.NoError(err)
-		})
-	})
 }
 
 func TestConcurrent(t *testing.T) {
@@ -498,39 +468,63 @@ func TestLocking(t *testing.T) {
 }
 
 type benchmarkTestCase struct {
-	name string
+	name     string
-	size int
+	size     int
+	withPool bool
 }
 
 func BenchmarkGet(b *testing.B) {
-	testdir, err := ioutil.TempDir("", "bitcask")
+	currentDir, err := os.Getwd()
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	db, err := Open(testdir)
+	testdir, err := ioutil.TempDir(currentDir, "bitcask_bench")
 	if err != nil {
 		b.Fatal(err)
 	}
-	defer db.Close()
+	defer os.RemoveAll(testdir)
 
 	tests := []benchmarkTestCase{
-		{"128B", 128},
+		{"128B", 128, false},
-		{"256B", 256},
+		{"128BWithPool", 128, true},
-		{"512B", 512},
+		{"256B", 256, false},
-		{"1K", 1024},
+		{"256BWithPool", 256, true},
-		{"2K", 2048},
+		{"512B", 512, false},
-		{"4K", 4096},
+		{"512BWithPool", 512, true},
-		{"8K", 8192},
+		{"1K", 1024, false},
-		{"16K", 16384},
+		{"1KWithPool", 1024, true},
-		{"32K", 32768},
+		{"2K", 2048, false},
+		{"2KWithPool", 2048, true},
+		{"4K", 4096, false},
+		{"4KWithPool", 4096, true},
+		{"8K", 8192, false},
+		{"8KWithPool", 8192, true},
+		{"16K", 16384, false},
+		{"16KWithPool", 16384, true},
+		{"32K", 32768, false},
+		{"32KWithPool", 32768, true},
 	}
 
 	for _, tt := range tests {
 		b.Run(tt.name, func(b *testing.B) {
+			b.SetBytes(int64(tt.size))
+
 			key := "foo"
 			value := []byte(strings.Repeat(" ", tt.size))
 
+			options := []Option{
+				WithMaxKeySize(len(key)),
+				WithMaxValueSize(tt.size),
+			}
+			if tt.withPool {
+				options = append(options, WithMemPool(1))
+			}
+			db, err := Open(testdir, options...)
+			if err != nil {
+				b.Fatal(err)
+			}
+
 			err = db.Put(key, value)
 			if err != nil {
 				b.Fatal(err)
@@ -546,16 +540,24 @@ func BenchmarkGet(b *testing.B) {
 					b.Errorf("unexpected value")
 				}
 			}
+			b.StopTimer()
+			db.Close()
 		})
 	}
 }
 
 func BenchmarkPut(b *testing.B) {
-	testdir, err := ioutil.TempDir("", "bitcask")
+	currentDir, err := os.Getwd()
 	if err != nil {
 		b.Fatal(err)
 	}
 
+	testdir, err := ioutil.TempDir(currentDir, "bitcask_bench")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer os.RemoveAll(testdir)
+
 	db, err := Open(testdir)
 	if err != nil {
 		b.Fatal(err)
@@ -563,19 +565,21 @@ func BenchmarkPut(b *testing.B) {
 	defer db.Close()
 
 	tests := []benchmarkTestCase{
-		{"128B", 128},
+		{"128B", 128, false},
-		{"256B", 256},
+		{"256B", 256, false},
-		{"512B", 512},
+		{"512B", 512, false},
-		{"1K", 1024},
+		{"1K", 1024, false},
-		{"2K", 2048},
+		{"2K", 2048, false},
-		{"4K", 4096},
+		{"4K", 4096, false},
-		{"8K", 8192},
+		{"8K", 8192, false},
-		{"16K", 16384},
+		{"16K", 16384, false},
-		{"32K", 32768},
+		{"32K", 32768, false},
 	}
 
 	for _, tt := range tests {
 		b.Run(tt.name, func(b *testing.B) {
+			b.SetBytes(int64(tt.size))
+
 			key := "foo"
 			value := []byte(strings.Repeat(" ", tt.size))
 			b.ResetTimer()
@@ -590,11 +594,17 @@ func BenchmarkPut(b *testing.B) {
 }
 
 func BenchmarkScan(b *testing.B) {
-	testdir, err := ioutil.TempDir("", "bitcask")
+	currentDir, err := os.Getwd()
 	if err != nil {
 		b.Fatal(err)
 	}
 
+	testdir, err := ioutil.TempDir(currentDir, "bitcask_bench")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer os.RemoveAll(testdir)
+
 	db, err := Open(testdir)
 	if err != nil {
 		b.Fatal(err)
@@ -626,6 +636,9 @@ func BenchmarkScan(b *testing.B) {
 			keys = append(keys, key)
 			return nil
 		})
+		if err != nil {
+			b.Fatal(err)
+		}
 		sort.Strings(keys)
 		if !reflect.DeepEqual(expected, keys) {
 			b.Fatal(fmt.Errorf("expected keys=#%v got=%#v", expected, keys))

cmd/bitcask/merge.go

@@ -20,28 +20,23 @@ keys.`,
 	Args: cobra.ExactArgs(0),
 	Run: func(cmd *cobra.Command, args []string) {
 		path := viper.GetString("path")
-		force, err := cmd.Flags().GetBool("force")
-		if err != nil {
-			log.WithError(err).Error("error parsing force flag")
-			os.Exit(1)
-		}
 
-		os.Exit(merge(path, force))
+		os.Exit(merge(path))
 	},
 }
 
 func init() {
 	RootCmd.AddCommand(mergeCmd)
-
-	mergeCmd.Flags().BoolP(
-		"force", "f", false,
-		"Force a re-merge even if .hint files exist",
-	)
 }
 
-func merge(path string, force bool) int {
+func merge(path string) int {
-	err := bitcask.Merge(path, force)
+	db, err := bitcask.Open(path)
 	if err != nil {
+		log.WithError(err).Error("error opening database")
+		return 1
+	}
+
+	if err = db.Merge(); err != nil {
 		log.WithError(err).Error("error merging database")
 		return 1
 	}

cmd/bitcask/stats.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"github.com/prologic/bitcask"
+)
+
+var statsCmd = &cobra.Command{
+	Use:     "stats",
+	Aliases: []string{},
+	Short:   "Display statis about the Database",
+	Long:    `This displays statistics about the Database"`,
+	Args:    cobra.ExactArgs(0),
+	Run: func(cmd *cobra.Command, args []string) {
+		path := viper.GetString("path")
+
+		os.Exit(stats(path))
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(statsCmd)
+}
+
+func stats(path string) int {
+	db, err := bitcask.Open(path)
+	if err != nil {
+		log.WithError(err).Error("error opening database")
+		return 1
+	}
+	defer db.Close()
+
+	stats, err := db.Stats()
+	if err != nil {
+		log.WithError(err).Error("error getting stats")
+		return 1
+	}
+
+	data, err := json.MarshalIndent(stats, "", "  ")
+	if err != nil {
+		log.WithError(err).Error("error marshalling stats")
+		return 1
+	}
+
+	fmt.Println(string(data))
+
+	return 0
+}

doc.go

@@ -0,0 +1,13 @@
+// Package bitcask implements a high-performance key-value store based on a
+// WAL and LSM.
+//
+// By default, the client assumes a default configuration regarding maximum key size,
+// maximum value size, maximum datafile size, and memory pools to avoid allocations.
+// Refer to Constants section to know default values.
+//
+// For extra performance, configure the memory pool option properly. This option
+// requires to specify the maximum number of concurrent use of the package. Failing to
+// set a high-enough value would impact latency and throughput. Likewise, overestimating
+// would yield in an unnecessary big memory footprint.
+// The default configuration doesn't use a memory pool.
+package bitcask

doc_test.go

@@ -0,0 +1,14 @@
+package bitcask
+
+func Example() {
+	_, _ = Open("path/to/db")
+}
+
+func Example_withOptions() {
+	opts := []Option{
+		WithMaxKeySize(1024),
+		WithMaxValueSize(4096),
+		WithMemPool(10),
+	}
+	_, _ = Open("path/to/db", opts...)
+}

go.mod

@@ -1,29 +1,24 @@
 module github.com/prologic/bitcask
 
 require (
-	github.com/BurntSushi/toml v0.3.1 // indirect
-	github.com/coreos/etcd v3.3.12+incompatible // indirect
 	github.com/gofrs/flock v0.7.1
 	github.com/gogo/protobuf v1.2.1
-	github.com/golang/protobuf v1.3.1
+	github.com/golang/protobuf v1.3.2
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/kisielk/errcheck v1.2.0 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/magiconair/properties v1.8.1 // indirect
+	github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c
+	github.com/pelletier/go-toml v1.4.0 // indirect
 	github.com/pkg/errors v0.8.1
 	github.com/prologic/trie v0.0.0-20190322091023-3972df81f9b5
-	github.com/sirupsen/logrus v1.4.0
+	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/afero v1.2.1 // indirect
+	github.com/spf13/afero v1.2.2 // indirect
-	github.com/spf13/cobra v0.0.3
+	github.com/spf13/cobra v0.0.5
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.3
-	github.com/spf13/viper v1.3.2
+	github.com/spf13/viper v1.4.0
 	github.com/stretchr/testify v1.3.0
 	github.com/tidwall/redcon v1.0.0
-	github.com/ugorji/go/codec v0.0.0-20190320090025-2dc34c0b8780 // indirect
+	golang.org/x/exp v0.0.0-20190718202018-cfdd5522f6f6
-	golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576 // indirect
+	golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 // indirect
-	golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53 // indirect
+	golang.org/x/text v0.3.2 // indirect
-	golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 // indirect
-	golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc // indirect
-	golang.org/x/tools v0.0.0-20190321232350-e250d351ecad // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 )

go.sum

@@ -1,107 +1,200 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gofrs/flock v0.7.1 h1:DP+LD/t0njgoPBvT5MJLeliUIVQR03hiKR6vezdwHlc=
 github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
-github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw=
+github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg=
+github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prologic/trie v0.0.0-20190316011403-395e39dac705 h1:2J+cSlAeECj0lfMKSmM7n5OlIio+yLovaKLZJzwLc6U=
-github.com/prologic/trie v0.0.0-20190316011403-395e39dac705/go.mod h1:LFuDmpHJGmciXd8Rl5YMhVlLMps9gz2GtYLzwxrFhzs=
 github.com/prologic/trie v0.0.0-20190322091023-3972df81f9b5 h1:H8dTZzU3aWNQnuRyiT45J9szv7EFakAhFzsFq27t3Uo=
 github.com/prologic/trie v0.0.0-20190322091023-3972df81f9b5/go.mod h1:LFuDmpHJGmciXd8Rl5YMhVlLMps9gz2GtYLzwxrFhzs=
-github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
-github.com/sirupsen/logrus v1.4.0 h1:yKenngtzGh+cUSSh6GWbxW2abRqhYUSR/t/6+2QqNvE=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
-github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
+github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
-github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38=
-github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
+github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/tidwall/redcon v0.9.0 h1:tiT9DLAoohsdNaFg9Si5dRsv9+FjvZYnhMOEtSFwBqA=
-github.com/tidwall/redcon v0.9.0/go.mod h1:bdYBm4rlcWpst2XMwKVzWDF9CoUxEbUmM7CQrKeOZas=
 github.com/tidwall/redcon v1.0.0 h1:D4AzzJ81Afeh144fgnj5H0aSVPBBJ5RI9Rzj0zThU+E=
 github.com/tidwall/redcon v1.0.0/go.mod h1:bdYBm4rlcWpst2XMwKVzWDF9CoUxEbUmM7CQrKeOZas=
-github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
-github.com/ugorji/go/codec v0.0.0-20190320090025-2dc34c0b8780/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576 h1:aUX/1G2gFSs4AsJJg2cL3HuoRhCSCz733FE5GUSuaT4=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/exp v0.0.0-20190718202018-cfdd5522f6f6 h1:1xlTfQXUyQvvbx4TFsXRgoj9r0YUwcqq9XGX9u9OODU=
+golang.org/x/exp v0.0.0-20190718202018-cfdd5522f6f6/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc h1:4gbWbmmPFp4ySWICouJl6emP0MyS31yy9SrTlAGFT+g=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI=
+golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190321232350-e250d351ecad/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

internal/datafile.go

@@ -1,26 +1,31 @@
 package internal
 
 import (
-	"bytes"
 	"fmt"
 	"os"
 	"path/filepath"
 	"sync"
 
+	"github.com/oxtoacart/bpool"
 	"github.com/pkg/errors"
-	log "github.com/sirupsen/logrus"
+	"golang.org/x/exp/mmap"
 
+	"github.com/gogo/protobuf/proto"
 	pb "github.com/prologic/bitcask/internal/proto"
 	"github.com/prologic/bitcask/internal/streampb"
 )
 
 const (
 	DefaultDatafileFilename = "%09d.data"
+	prefixSize              = 8
 )
 
 var (
 	ErrReadonly  = errors.New("error: read only datafile")
 	ErrReadError = errors.New("error: read error")
+
+	memPool   *bpool.BufferPool
+	mxMemPool sync.RWMutex
 )
 
 type Datafile struct {
@@ -28,6 +33,7 @@ type Datafile struct {
 
 	id     int
 	r      *os.File
+	ra     *mmap.ReaderAt
 	w      *os.File
 	offset int64
 	dec    *streampb.Decoder
@@ -37,6 +43,7 @@ type Datafile struct {
 func NewDatafile(path string, id int, readonly bool) (*Datafile, error) {
 	var (
 		r   *os.File
+		ra  *mmap.ReaderAt
 		w   *os.File
 		err error
 	)
@@ -59,6 +66,11 @@ func NewDatafile(path string, id int, readonly bool) (*Datafile, error) {
 		return nil, errors.Wrap(err, "error calling Stat()")
 	}
 
+	ra, err = mmap.Open(fn)
+	if err != nil {
+		return nil, err
+	}
+
 	offset := stat.Size()
 
 	dec := streampb.NewDecoder(r)
@@ -67,6 +79,7 @@ func NewDatafile(path string, id int, readonly bool) (*Datafile, error) {
 	return &Datafile{
 		id:     id,
 		r:      r,
+		ra:     ra,
 		w:      w,
 		offset: offset,
 		dec:    dec,
@@ -83,8 +96,14 @@ func (df *Datafile) Name() string {
 }
 
 func (df *Datafile) Close() error {
+	defer func() {
+		df.ra.Close()
+		df.r.Close()
+	}()
+
+	// Readonly Datafile -- Nothing further to close on the write side
 	if df.w == nil {
-		return df.r.Close()
+		return nil
 	}
 
 	err := df.Sync()
@@ -120,10 +139,25 @@ func (df *Datafile) Read() (e pb.Entry, n int64, err error) {
 }
 
 func (df *Datafile) ReadAt(index, size int64) (e pb.Entry, err error) {
-	log.WithField("index", index).WithField("size", size).Debug("ReadAt")
+	var n int
 
-	b := make([]byte, size)
+	var b []byte
-	n, err := df.r.ReadAt(b, index)
+	if memPool == nil {
+		b = make([]byte, size)
+	} else {
+		poolSlice := memPool.Get()
+		if poolSlice.Cap() < int(size) {
+			poolSlice.Grow(int(size) - poolSlice.Cap())
+		}
+		defer memPool.Put(poolSlice)
+		b = poolSlice.Bytes()[:size]
+	}
+
+	if df.w == nil {
+		n, err = df.ra.ReadAt(b, index)
+	} else {
+		n, err = df.r.ReadAt(b, index)
+	}
 	if err != nil {
 		return
 	}
@@ -132,9 +166,10 @@ func (df *Datafile) ReadAt(index, size int64) (e pb.Entry, err error) {
 		return
 	}
 
-	buf := bytes.NewBuffer(b)
+	err = proto.Unmarshal(b[prefixSize:], &e)
-	dec := streampb.NewDecoder(buf)
+	if err != nil {
-	_, err = dec.Decode(&e)
+		return
+	}
 	return
 }
 
@@ -156,3 +191,15 @@ func (df *Datafile) Write(e pb.Entry) (int64, int64, error) {
 
 	return e.Offset, n, nil
 }
+
+// ConfigureMemPool configurate the mempool accordingly
+func ConfigureMemPool(maxConcurrency *int) {
+	mxMemPool.Lock()
+	defer mxMemPool.Unlock()
+	if maxConcurrency == nil {
+		memPool = nil
+	} else {
+		memPool = bpool.NewBufferPool(*maxConcurrency)
+	}
+	return
+}

internal/keydir.go

@@ -5,6 +5,7 @@ import (
 	"encoding/gob"
 	"io"
 	"io/ioutil"
+	"os"
 	"sync"
 )
 
@@ -41,17 +42,15 @@ func (k *Keydir) Add(key string, fileid int, offset, size int64) Item {
 
 func (k *Keydir) Get(key string) (Item, bool) {
 	k.RLock()
-	defer k.RUnlock()
-
 	item, ok := k.kv[key]
+	k.RUnlock()
 	return item, ok
 }
 
 func (k *Keydir) Delete(key string) {
 	k.Lock()
-	defer k.Unlock()
-
 	delete(k.kv, key)
+	k.Unlock()
 }
 
 func (k *Keydir) Len() int {
@@ -62,11 +61,11 @@ func (k *Keydir) Keys() chan string {
 	ch := make(chan string)
 	go func() {
 		k.RLock()
-		defer k.RUnlock()
 		for key := range k.kv {
 			ch <- key
 		}
 		close(ch)
+		k.RUnlock()
 	}()
 	return ch
 }
@@ -81,6 +80,21 @@ func (k *Keydir) Bytes() ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
+func (k *Keydir) Load(fn string) error {
+	f, err := os.Open(fn)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	dec := gob.NewDecoder(f)
+	if err := dec.Decode(&k.kv); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (k *Keydir) Save(fn string) error {
 	data, err := k.Bytes()
 	if err != nil {

internal/proto/entry.pb.go

@@ -32,7 +32,7 @@ func (m *Entry) Reset()         { *m = Entry{} }
 func (m *Entry) String() string { return proto.CompactTextString(m) }
 func (*Entry) ProtoMessage()    {}
 func (*Entry) Descriptor() ([]byte, []int) {
-	return fileDescriptor_entry_3e91842c99935ae2, []int{0}
+	return fileDescriptor_entry_db5b99f271e6b4b6, []int{0}
 }
 func (m *Entry) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_Entry.Unmarshal(m, b)
@@ -84,9 +84,9 @@ func init() {
 	proto.RegisterType((*Entry)(nil), "proto.Entry")
 }
 
-func init() { proto.RegisterFile("entry.proto", fileDescriptor_entry_3e91842c99935ae2) }
+func init() { proto.RegisterFile("entry.proto", fileDescriptor_entry_db5b99f271e6b4b6) }
 
-var fileDescriptor_entry_3e91842c99935ae2 = []byte{
+var fileDescriptor_entry_db5b99f271e6b4b6 = []byte{
 	// 126 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4e, 0xcd, 0x2b, 0x29,
 	0xaa, 0xd4, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x05, 0x53, 0x4a, 0xc9, 0x5c, 0xac, 0xae,

internal/utils.go

@@ -2,12 +2,32 @@ package internal
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
 )
 
+func Exists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+func DirSize(path string) (int64, error) {
+	var size int64
+	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			size += info.Size()
+		}
+		return err
+	})
+	return size, err
+}
+
 func GetDatafiles(path string) ([]string, error) {
 	fns, err := filepath.Glob(fmt.Sprintf("%s/*.data", path))
 	if err != nil {

options.go

@@ -1,5 +1,7 @@
 package bitcask
 
+import "errors"
+
 const (
 	// DefaultMaxDatafileSize is the default maximum datafile size in bytes
 	DefaultMaxDatafileSize = 1 << 20 // 1MB
@@ -11,6 +13,12 @@ const (
 	DefaultMaxValueSize = 1 << 16 // 65KB
 )
 
+var (
+	// ErrMaxConcurrencyLowerEqZero is the error returned for
+	// maxConcurrency option not greater than zero
+	ErrMaxConcurrencyLowerEqZero = errors.New("error: maxConcurrency must be greater than zero")
+)
+
 // Option is a function that takes a config struct and modifies it
 type Option func(*config) error
 
@@ -18,6 +26,7 @@ type config struct {
 	maxDatafileSize int
 	maxKeySize      int
 	maxValueSize    int
+	maxConcurrency  *int
 }
 
 func newDefaultConfig() *config {
@@ -51,3 +60,14 @@ func WithMaxValueSize(size int) Option {
 		return nil
 	}
 }
+
+// WithMemPool configures usage of a memory pool to avoid allocations
+func WithMemPool(maxConcurrency int) Option {
+	return func(cfg *config) error {
+		if maxConcurrency <= 0 {
+			return ErrMaxConcurrencyLowerEqZero
+		}
+		cfg.maxConcurrency = &maxConcurrency
+		return nil
+	}
+}

tools/release.sh

@@ -20,6 +20,6 @@ fi
 
 echo "Releasing ${TAG} ..."
 
-git tag -a -s -m "Relase ${TAG}" "${TAG}"
+git tag -a -s -m "Release ${TAG}" "${TAG}"
 git push --tags
 goreleaser release --rm-dist