Update CHANGELOG for v0.3.10

Fix a bug when MaxValueSize == 0 on Merge operations
Save metadata on Sync (#197 )
2026-04-02 11:09:01 -07:00 · 2020-12-18 23:18:50 +10:00 · 2020-12-18 07:35:16 +10:00 · 2020-12-15 06:32:48 +10:00 · 2020-12-13 21:28:54 +10:00 · 2020-12-12 07:55:29 +10:00
51 changed files with 4730 additions and 880 deletions
--- a/.chglog/CHANGELOG.tpl.md
+++ b/.chglog/CHANGELOG.tpl.md
@@ -0,0 +1,22 @@
+{{ range .Versions }}
+<a name="{{ .Tag.Name }}"></a>
+## {{ if .Tag.Previous }}[{{ .Tag.Name }}]({{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}){{ else }}{{ .Tag.Name }}{{ end }} ({{ datetime "2006-01-02" .Tag.Date }})
+
+{{ range .CommitGroups -}}
+### {{ .Title }}
+
+{{ range .Commits -}}
+* {{ .Subject }}
+{{ end }}
+{{ end -}}
+
+{{- if .NoteGroups -}}
+{{ range .NoteGroups -}}
+### {{ .Title }}
+
+{{ range .Notes }}
+{{ .Body }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+{{ end -}}
--- a/.chglog/config.yml
+++ b/.chglog/config.yml
@@ -0,0 +1,28 @@
+---
+style: Github
+template: CHANGELOG.tpl.md
+info:
+  title: CHANGELOG
+  repository_url: https://github.com/prologic/bitcask
+options:
+  commits:
+    filters:
+      Type:
+        - Add
+        - Fix
+        - Update
+        - Document
+  commit_groups:
+    title_maps:
+      Add: Features
+      Update: Updates
+      Fix: Bug Fixes
+      Document: Documentation
+  header:
+    pattern: "^((\\w+)\\s.*)$"
+    pattern_maps:
+      - Subject
+      - Type
+  notes:
+    keywords:
+      - BREAKING CHANGE
--- a/.dependabot/config.yml
+++ b/.dependabot/config.yml
@@ -0,0 +1,8 @@
+version: 1
+update_configs:
+  - package_manager: "go:modules"
+    directory: "/"
+    update_schedule: "daily"
+  - package_manager: "docker"
+    directory: "/"
+    update_schedule: "weekly"
--- a/.drone.yml
+++ b/.drone.yml
@@ -1,14 +0,0 @@
-kind: pipeline
-name: default
-
-steps:
-  - name: build
-    image: golang:latest
-    commands:
-      - go test -v -short -cover -coverprofile=coverage.txt -coverpkg=$(go list) .
-
-  - name: coverage
-    image: plugins/codecov
-    settings:
-      token:
-        from_secret: codecov-token
--- a/.github/labeler.yml
+++ b/.github/labeler.yml
@@ -0,0 +1,9 @@
+documentation:
+  - "**/*.md"
+
+tests:
+  - "**/*_test.go"
+
+dependencies:
+  - go.mod
+  - go.sum
--- a/.github/workflows/codecov.yml
+++ b/.github/workflows/codecov.yml
@@ -0,0 +1,23 @@
+name: Coverage
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+jobs:
+  test:
+    name: Test and Report
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13.x
+      - name: Checkout
+        uses: actions/checkout@v1
+      - name: Test
+        run: go test -v -cover -coverprofile=coverage.txt -covermode=atomic -race .
+      - name: Report
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -0,0 +1,15 @@
+name: Docker
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+jobs:
+  build:
+    name: Build Image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+      - name: Build
+        run: docker build -t bitcask .
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -0,0 +1,27 @@
+---
+name: Go
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+jobs:
+  test:
+    name: Build and Test
+    strategy:
+      matrix:
+        go-version: [1.12.x, 1.13.x, 1.14.x, 1.15.x]
+        platform: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - name: Setup Go ${{ matrix.go-version }}
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+        id: go
+      - name: Checkout
+        uses: actions/checkout@v1
+      - name: Build
+        run: go build -v .
+      - name: Test
+        run: go test -v -race .
--- a/.github/workflows/reviewdog.yml
+++ b/.github/workflows/reviewdog.yml
@@ -0,0 +1,32 @@
+name: ReviewDog
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+jobs:
+  golangci-lint:
+    name: runner / golangci-lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: reviewdog/action-golangci-lint@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+  misspell:
+      name: runner / misspell
+      runs-on: ubuntu-latest
+      steps:
+        - uses: actions/checkout@v1
+        - uses: reviewdog/action-misspell@v1
+          with:
+            github_token: ${{ secrets.GItHUB_TOKEN }}
+  shellcheck:
+      name: runner / shellcheck
+      runs-on: ubuntu-latest
+      steps:
+        - uses: actions/checkout@v1
+        - uses: reviewdog/action-shellcheck@v1
+          with:
+            github_token: ${{ secrets.GITHUB_TOKEN }}
+            reporter: github-pr-review
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -0,0 +1,29 @@
+name: Mark stale issues and pull requests
+
+on:
+  schedule:
+  - cron: "0 0 * * *"
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/stale@v1
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: |-
+          This Issue has gone stale and will be closed automatically.
+          
+          If this is incorrect, please reopen and add the label `on hold`.
+          
+          Thank you.
+        stale-pr-message: |-
+          This Pull Request has gone stale and will be closed automatically.
+          
+          If this is incorrect, please reopen and add the label `on hold`.
+          
+          Thank you.
+        stale-issue-label: 'stale'
+        exempt-issue-label: 'on hold'
+        stale-pr-label: 'stale'
+        exempt-pr-label: 'on hold'
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,13 @@
 *~*
 *.bak
+*.db

-/coverage.txt
-/bitcask
-/bitcaskd
 /tmp
 /dist
+/coverage.txt
+
+/bitcask
+/bitcaskd
+/bitcask_bench*
+/cmd/bitcask/bitcask
+/cmd/bitcaskd/bitcaskd
--- a/.goreleaser.yml
+++ b/.goreleaser.yml
@@ -1,34 +1,25 @@
+---
 builds:
-  - 
+  -
+    id: bitcask
    binary: bitcask
    main: ./cmd/bitcask
    flags: -tags "static_build"
    ldflags: -w -X github.com/prologic/bitcask/internal.Version={{.Version}} -X github.com/prologic/bitcask/internal.Commit={{.Commit}}
    env:
      - CGO_ENABLED=0
-  - 
+  -
+    id: bitcaskd
    binary: bitcaskd
    main: ./cmd/bitcaskd
    flags: -tags "static_build"
    ldflags: -w -X github.com/prologic/bitcask/internal.Version={{.Version}} -X github.com/prologic/bitcask/internal.Commit={{.Commit}}
    env:
      - CGO_ENABLED=0
-sign:
-  artifacts: checksum
-archive:
-  replacements:
-    darwin: Darwin
-    linux: Linux
-    windows: Windows
-    386: i386
-    amd64: x86_64
-checksum:
-  name_template: 'checksums.txt'
-snapshot:
-  name_template: "{{ .Tag }}-next"
-changelog:
-  sort: asc
-  filters:
-    exclude:
-    - '^docs:'
-    - '^test:'
+signs:
+  - artifacts: checksum
+release:
+  github:
+    owner: prologic
+    name: bitcask
+  draft: true
--- a/5
+++ b/5
@@ -2,12 +2,17 @@
 #   Name or Organization <email address>
 # The email address is not required for organizations.

+Alain Gilbert <alain.gilbert.15@gmail.com>
 Awn Umar <awn@spacetime.dev>
+Bryan Stenson <bryan@siliconvortex.com>
 Christian Muehlhaeuser <muesli@gmail.com>
 Ignacio Hagopian <jsign.uy@gmail.com>
 James Mills <prologic@shortcircuit.net.au>
 Jesse Donat <donatj@gmail.com>
 Kebert Xela kebertxela
 panyun panyun 
+shiniao <zhuzhezhe95@gmail.com>
 Whemoon Jang <palindrom615@gmail.com>
 Yury Fedorov orlangure
+o2gy84 <o2gy84@gmail.com>
+garsue <labs.garsue@gmail.com>
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -0,0 +1,394 @@
+
+<a name="v0.3.10"></a>
+## [v0.3.10](https://github.com/prologic/bitcask/compare/v0.3.9...v0.3.10) (2020-12-18)
+
+### Bug Fixes
+
+* Fix a bug when MaxValueSize == 0 on Merge operations
+* Fix link to bitcask-bench
+* Fix CI (again)
+* Fix CI
+
+### Features
+
+* Add support for unlimited key/value sizes
+* Add a few more test cases for concurrent operations
+
+### Updates
+
+* Update README.md
+
+
+<a name="v0.3.9"></a>
+## [v0.3.9](https://github.com/prologic/bitcask/compare/v0.3.8...v0.3.9) (2020-11-17)
+
+### Bug Fixes
+
+* Fix a race condition around .Close() and .Sync()
+
+### Updates
+
+* Update CHANGELOG for v0.3.9
+
+
+<a name="v0.3.8"></a>
+## [v0.3.8](https://github.com/prologic/bitcask/compare/v0.3.7...v0.3.8) (2020-11-17)
+
+### Updates
+
+* Update CHANGELOG for v0.3.8
+
+
+<a name="v0.3.7"></a>
+## [v0.3.7](https://github.com/prologic/bitcask/compare/v0.3.6...v0.3.7) (2020-11-17)
+
+### Updates
+
+* Update CHANGELOG for v0.3.7
+
+
+<a name="v0.3.6"></a>
+## [v0.3.6](https://github.com/prologic/bitcask/compare/v0.3.5...v0.3.6) (2020-11-17)
+
+### Bug Fixes
+
+* Fix typo in labeler ([#172](https://github.com/prologic/bitcask/issues/172))
+* Fix builds configuration for goreleaser
+* Fix (again) goreleaser config
+* Fix goreleaser config and improve release notes / changelog
+* Fix recoverDatafile error covering ([#162](https://github.com/prologic/bitcask/issues/162))
+* Fix loadIndex to be deterministic ([#115](https://github.com/prologic/bitcask/issues/115))
+
+### Features
+
+* Add configuration options for FileMode ([#183](https://github.com/prologic/bitcask/issues/183))
+* Add imports and log in example code ([#182](https://github.com/prologic/bitcask/issues/182))
+* Add empty changelog
+* Add DependaBot config
+* Add DeleteAll function ([#116](https://github.com/prologic/bitcask/issues/116))
+
+### Updates
+
+* Update CHANGELOG for v0.3.6
+* Update README.md
+* Update CHANGELOG for v0.3.6
+* Update CHANGELOG for v0.3.6
+* Update deps ([#140](https://github.com/prologic/bitcask/issues/140))
+* Update README.md
+
+
+<a name="v0.3.5"></a>
+## [v0.3.5](https://github.com/prologic/bitcask/compare/v0.3.4...v0.3.5) (2019-10-20)
+
+### Bug Fixes
+
+* Fix setup target in Makefile to install mockery correctly
+* Fix glfmt/golint issues
+* Fix spelling mistake in README s/Sponser/Sponsor
+
+### Features
+
+* Add *.db to ignore future accidental commits of a bitcask db to the repo
+* Add unit test for opening bad database with corrupted/invalid datafiles ([#105](https://github.com/prologic/bitcask/issues/105))
+
+### Updates
+
+* Update Drone CI test pipeline
+* Update README.md
+* Update to Go 1.13 and update README with new benchmarks ([#89](https://github.com/prologic/bitcask/issues/89))
+* Update README.md
+
+
+<a name="v0.3.4"></a>
+## [v0.3.4](https://github.com/prologic/bitcask/compare/v0.3.3...v0.3.4) (2019-09-02)
+
+
+<a name="v0.3.3"></a>
+## [v0.3.3](https://github.com/prologic/bitcask/compare/v0.3.2...v0.3.3) (2019-09-02)
+
+### Bug Fixes
+
+* Fix a bug wit the decoder passing the wrong value for the value's offset into the buffer ([#77](https://github.com/prologic/bitcask/issues/77))
+* Fix typo ([#65](https://github.com/prologic/bitcask/issues/65))
+* Fix and cleanup some unnecessary internal sub-packages and duplication
+
+### Updates
+
+* Update README.md
+* Update README.md
+* Update README.md
+* Update README.md
+
+
+<a name="v0.3.2"></a>
+## [v0.3.2](https://github.com/prologic/bitcask/compare/v0.3.1...v0.3.2) (2019-08-08)
+
+### Updates
+
+* Update README.md
+* Update README.md
+* Update CONTRIBUTING.md
+
+
+<a name="v0.3.1"></a>
+## [v0.3.1](https://github.com/prologic/bitcask/compare/v0.3.0...v0.3.1) (2019-08-05)
+
+### Updates
+
+* Update README.md
+* Update README.md
+* Update README.md
+
+
+<a name="v0.3.0"></a>
+## [v0.3.0](https://github.com/prologic/bitcask/compare/v0.2.2...v0.3.0) (2019-07-29)
+
+### Updates
+
+* Update README.md
+* Update README.md
+
+
+<a name="v0.2.2"></a>
+## [v0.2.2](https://github.com/prologic/bitcask/compare/v0.2.1...v0.2.2) (2019-07-27)
+
+
+<a name="v0.2.1"></a>
+## [v0.2.1](https://github.com/prologic/bitcask/compare/v0.2.0...v0.2.1) (2019-07-25)
+
+
+<a name="v0.2.0"></a>
+## [v0.2.0](https://github.com/prologic/bitcask/compare/v0.1.7...v0.2.0) (2019-07-25)
+
+### Bug Fixes
+
+* Fix issue(db file Merge issue in windows env): ([#15](https://github.com/prologic/bitcask/issues/15))
+
+
+<a name="v0.1.7"></a>
+## [v0.1.7](https://github.com/prologic/bitcask/compare/v0.1.6...v0.1.7) (2019-07-19)
+
+### Bug Fixes
+
+* Fix mismatched key casing. ([#12](https://github.com/prologic/bitcask/issues/12))
+* Fix outdated README ([#11](https://github.com/prologic/bitcask/issues/11))
+* Fix typos in bitcask.go docs ([#10](https://github.com/prologic/bitcask/issues/10))
+
+### Updates
+
+* Update generated protobuf code
+* Update README.md
+
+
+<a name="v0.1.6"></a>
+## [v0.1.6](https://github.com/prologic/bitcask/compare/v0.1.5...v0.1.6) (2019-04-01)
+
+### Features
+
+* Add Development section to README documenting use of Protobuf and tooling required. [#6](https://github.com/prologic/bitcask/issues/6)
+* Add other badges from img.shields.io
+
+
+<a name="v0.1.5"></a>
+## [v0.1.5](https://github.com/prologic/bitcask/compare/v0.1.4...v0.1.5) (2019-03-30)
+
+### Documentation
+
+* Document using the Docker Image
+
+### Features
+
+* Add Dockerfile to publish images to Docker Hub
+
+### Updates
+
+* Update README.md
+
+
+<a name="v0.1.4"></a>
+## [v0.1.4](https://github.com/prologic/bitcask/compare/v0.1.3...v0.1.4) (2019-03-23)
+
+
+<a name="v0.1.3"></a>
+## [v0.1.3](https://github.com/prologic/bitcask/compare/v0.1.2...v0.1.3) (2019-03-23)
+
+
+<a name="v0.1.2"></a>
+## [v0.1.2](https://github.com/prologic/bitcask/compare/v0.1.1...v0.1.2) (2019-03-22)
+
+
+<a name="v0.1.1"></a>
+## [v0.1.1](https://github.com/prologic/bitcask/compare/v0.1.0...v0.1.1) (2019-03-22)
+
+
+<a name="v0.1.0"></a>
+## [v0.1.0](https://github.com/prologic/bitcask/compare/0.0.26...v0.1.0) (2019-03-22)
+
+
+<a name="0.0.26"></a>
+## [0.0.26](https://github.com/prologic/bitcask/compare/0.0.25...0.0.26) (2019-03-21)
+
+### Features
+
+* Add docs for bitcask
+* Add docs for options
+* Add KeYS command to server (bitraftd)
+* Add Len() to exported API (extended API)
+* Add Keys() to exported API (extended API)
+* Add EXISTS command to server (bitraftd)
+
+
+<a name="0.0.25"></a>
+## [0.0.25](https://github.com/prologic/bitcask/compare/0.0.24...0.0.25) (2019-03-21)
+
+### Features
+
+* Add Has() to exported API (extended API)
+* Add MergeOpen test case
+
+### Updates
+
+* Update README.md
+* Update README.md
+
+
+<a name="0.0.24"></a>
+## [0.0.24](https://github.com/prologic/bitcask/compare/0.0.23...0.0.24) (2019-03-20)
+
+
+<a name="0.0.23"></a>
+## [0.0.23](https://github.com/prologic/bitcask/compare/0.0.22...0.0.23) (2019-03-20)
+
+### Features
+
+* Add bitcaskd to install target
+
+
+<a name="0.0.22"></a>
+## [0.0.22](https://github.com/prologic/bitcask/compare/0.0.21...0.0.22) (2019-03-18)
+
+
+<a name="0.0.21"></a>
+## [0.0.21](https://github.com/prologic/bitcask/compare/0.0.20...0.0.21) (2019-03-18)
+
+
+<a name="0.0.20"></a>
+## [0.0.20](https://github.com/prologic/bitcask/compare/0.0.19...0.0.20) (2019-03-17)
+
+
+<a name="0.0.19"></a>
+## [0.0.19](https://github.com/prologic/bitcask/compare/0.0.18...0.0.19) (2019-03-17)
+
+
+<a name="0.0.18"></a>
+## [0.0.18](https://github.com/prologic/bitcask/compare/0.0.17...0.0.18) (2019-03-16)
+
+
+<a name="0.0.17"></a>
+## [0.0.17](https://github.com/prologic/bitcask/compare/0.0.16...0.0.17) (2019-03-16)
+
+### Features
+
+* Add CRC Checksum checks on reading values back
+
+
+<a name="0.0.16"></a>
+## [0.0.16](https://github.com/prologic/bitcask/compare/0.0.15...0.0.16) (2019-03-16)
+
+
+<a name="0.0.15"></a>
+## [0.0.15](https://github.com/prologic/bitcask/compare/0.0.14...0.0.15) (2019-03-16)
+
+### Bug Fixes
+
+* Fix a race condition + Use my fork of trie
+
+
+<a name="0.0.14"></a>
+## [0.0.14](https://github.com/prologic/bitcask/compare/0.0.13...0.0.14) (2019-03-16)
+
+
+<a name="0.0.13"></a>
+## [0.0.13](https://github.com/prologic/bitcask/compare/0.0.12...0.0.13) (2019-03-16)
+
+### Features
+
+* Add prefix scan for keys using a Trie
+
+
+<a name="0.0.12"></a>
+## [0.0.12](https://github.com/prologic/bitcask/compare/0.0.11...0.0.12) (2019-03-14)
+
+
+<a name="0.0.11"></a>
+## [0.0.11](https://github.com/prologic/bitcask/compare/0.0.10...0.0.11) (2019-03-14)
+
+### Updates
+
+* Update README.md
+
+
+<a name="0.0.10"></a>
+## [0.0.10](https://github.com/prologic/bitcask/compare/0.0.9...0.0.10) (2019-03-14)
+
+### Bug Fixes
+
+* Fix concurrent read bug
+* Fix concurrent write bug with multiple goroutines writing to the to the active datafile
+
+### Updates
+
+* Update README.md
+
+
+<a name="0.0.9"></a>
+## [0.0.9](https://github.com/prologic/bitcask/compare/0.0.8...0.0.9) (2019-03-14)
+
+
+<a name="0.0.8"></a>
+## [0.0.8](https://github.com/prologic/bitcask/compare/0.0.7...0.0.8) (2019-03-13)
+
+
+<a name="0.0.7"></a>
+## [0.0.7](https://github.com/prologic/bitcask/compare/0.0.6...0.0.7) (2019-03-13)
+
+
+<a name="0.0.6"></a>
+## [0.0.6](https://github.com/prologic/bitcask/compare/0.0.5...0.0.6) (2019-03-13)
+
+### Bug Fixes
+
+* Fix usage output of bitcaskd
+
+
+<a name="0.0.5"></a>
+## [0.0.5](https://github.com/prologic/bitcask/compare/0.0.4...0.0.5) (2019-03-13)
+
+### Features
+
+* Add a simple Redis compatible server daemon (bitcaskd)
+
+### Updates
+
+* Update README.md
+
+
+<a name="0.0.4"></a>
+## [0.0.4](https://github.com/prologic/bitcask/compare/0.0.3...0.0.4) (2019-03-13)
+
+### Features
+
+* Add flock on database Open()/Close() to prevent multiple concurrent processes write access. Fixes [#2](https://github.com/prologic/bitcask/issues/2)
+
+
+<a name="0.0.3"></a>
+## [0.0.3](https://github.com/prologic/bitcask/compare/0.0.2...0.0.3) (2019-03-13)
+
+
+<a name="0.0.2"></a>
+## [0.0.2](https://github.com/prologic/bitcask/compare/0.0.1...0.0.2) (2019-03-13)
+
+
+<a name="0.0.1"></a>
+## 0.0.1 (2019-03-13)
+
--- a/8
+++ b/8
@@ -1,4 +1,4 @@
-.PHONY: dev build generate install image release profile bench test clean
+.PHONY: dev build generate install image release profile bench test clean setup

 CGO_ENABLED=0
 VERSION=$(shell git describe --abbrev=0 --tags)
@@ -39,6 +39,9 @@ profile: build
 bench: build
 	@go test -v -benchmem -bench=. .

+mocks:
+	@mockery -all -case underscore -output ./internal/mocks -recursive
+
 test: build
 	@go test -v \
 		-cover -coverprofile=coverage.txt -covermode=atomic \
@@ -46,5 +49,8 @@ test: build
 		-race \
 		.

+setup:
+	@go get github.com/vektra/mockery/...
+
 clean:
 	@git clean -f -d -X
--- a/README.md
+++ b/README.md
@@ -1,11 +1,17 @@
 # bitcask

-[![Build Status](https://cloud.drone.io/api/badges/prologic/bitcask/status.svg)](https://cloud.drone.io/prologic/bitcask)
+![](https://github.com/prologic/bitcask/workflows/Coverage/badge.svg)
+![](https://github.com/prologic/bitcask/workflows/Docker/badge.svg)
+![](https://github.com/prologic/bitcask/workflows/Go/badge.svg)
+![](https://github.com/prologic/bitcask/workflows/ReviewDog/badge.svg)
+
 [![CodeCov](https://codecov.io/gh/prologic/bitcask/branch/master/graph/badge.svg)](https://codecov.io/gh/prologic/bitcask)
 [![Go Report Card](https://goreportcard.com/badge/prologic/bitcask)](https://goreportcard.com/report/prologic/bitcask)
+[![codebeat badge](https://codebeat.co/badges/15fba8a5-3044-4f40-936f-9e0f5d5d1fd9)](https://codebeat.co/projects/github-com-prologic-bitcask-master)
 [![GoDoc](https://godoc.org/github.com/prologic/bitcask?status.svg)](https://godoc.org/github.com/prologic/bitcask) 
 [![GitHub license](https://img.shields.io/github/license/prologic/bitcask.svg)](https://github.com/prologic/bitcask)
 [![Sourcegraph](https://sourcegraph.com/github.com/prologic/bitcask/-/badge.svg)](https://sourcegraph.com/github.com/prologic/bitcask?badge)
+[![TODOs](https://img.shields.io/endpoint?url=https://api.tickgit.com/badge?repo=github.com/prologic/bitcask)](https://www.tickgit.com/browse?repo=github.com/prologic/bitcask)

 A high performance Key/Value store written in [Go](https://golang.org) with a predictable read/write performance and high throughput. Uses a [Bitcask](https://en.wikipedia.org/wiki/Bitcask) on-disk layout (LSM+WAL) similar to [Riak](https://riak.com/)

@@ -22,14 +28,14 @@ For a more feature-complete Redis-compatible server, distributed key/value store

 ## Development

-```#!sh
+```sh
 $ git clone https://github.com/prologic/bitcask.git
 $ make
 ```

 ## Install

-```#!sh
+```sh
 $ go get github.com/prologic/bitcask
 ```

@@ -37,20 +43,24 @@ $ go get github.com/prologic/bitcask

 Install the package into your project:

-```#!sh
+```sh
 $ go get github.com/prologic/bitcask
 ```

-```#!go
+```go
 package main

-import "github.com/prologic/bitcask"
+import (
+	"log"
+	"github.com/prologic/bitcask"
+)

 func main() {
    db, _ := bitcask.Open("/tmp/db")
    defer db.Close()
-    db.Put("Hello", []byte("World"))
-    val, _ := db.Get("Hello")
+    db.Put([]byte("Hello"), []byte("World"))
+    val, _ := db.Get([]byte("Hello"))
+    log.Printf(string(val))
 }
 ```

@@ -59,7 +69,7 @@ documentation and other examples.

 ## Usage (tool)

-```#!sh
+```sh
 $ bitcask -p /tmp/db set Hello World
 $ bitcask -p /tmp/db get Hello
 World
@@ -69,14 +79,14 @@ World

 There is also a builtin very  simple Redis-compatible server called `bitcaskd`:

-```#!sh
+```sh
 $ ./bitcaskd ./tmp
 INFO[0000] starting bitcaskd v0.0.7@146f777              bind=":6379" path=./tmp
 ```

 Example session:

-```#!sh
+```sh
 $ telnet localhost 6379
 Trying ::1...
 Connected to localhost.
@@ -101,7 +111,7 @@ Connection closed by foreign host.

 You can also use the [Bitcask Docker Image](https://cloud.docker.com/u/prologic/repository/docker/prologic/bitcask):

-```#!sh
+```sh
 $ docker pull prologic/bitcask
 $ docker run -d -p 6379:6379 prologic/bitcask
 ```
@@ -110,56 +120,52 @@ $ docker run -d -p 6379:6379 prologic/bitcask

 Benchmarks run on a 11" Macbook with a 1.4Ghz Intel Core i7:

-```#!sh
+```sh
 $ make bench
 ...
 goos: darwin
 goarch: amd64
 pkg: github.com/prologic/bitcask

-BenchmarkGet/128B-4         	  300000	      3913 ns/op	  32.71 MB/s	     387 B/op	       4 allocs/op
-BenchmarkGet/128BWithPool-4 	  300000	      4143 ns/op	  30.89 MB/s	     227 B/op	       3 allocs/op
-BenchmarkGet/256B-4         	  300000	      3919 ns/op	  65.31 MB/s	     643 B/op	       4 allocs/op
-BenchmarkGet/256BWithPool-4 	  300000	      4270 ns/op	  59.95 MB/s	     355 B/op	       3 allocs/op
-BenchmarkGet/512B-4         	  300000	      4248 ns/op	 120.52 MB/s	    1187 B/op	       4 allocs/op
-BenchmarkGet/512BWithPool-4 	  300000	      4676 ns/op	 109.48 MB/s	     611 B/op	       3 allocs/op
-BenchmarkGet/1K-4           	  200000	      5248 ns/op	 195.10 MB/s	    2275 B/op	       4 allocs/op
-BenchmarkGet/1KWithPool-4   	  200000	      5270 ns/op	 194.28 MB/s	    1123 B/op	       3 allocs/op
-BenchmarkGet/2K-4           	  200000	      6229 ns/op	 328.74 MB/s	    4451 B/op	       4 allocs/op
-BenchmarkGet/2KWithPool-4   	  200000	      6282 ns/op	 325.99 MB/s	    2147 B/op	       3 allocs/op
-BenchmarkGet/4K-4           	  200000	      9027 ns/op	 453.74 MB/s	    9059 B/op	       4 allocs/op
-BenchmarkGet/4KWithPool-4   	  200000	      8906 ns/op	 459.87 MB/s	    4195 B/op	       3 allocs/op
-BenchmarkGet/8K-4           	  100000	     12024 ns/op	 681.28 MB/s	   17763 B/op	       4 allocs/op
-BenchmarkGet/8KWithPool-4   	  200000	     11103 ns/op	 737.79 MB/s	    8291 B/op	       3 allocs/op
-BenchmarkGet/16K-4          	  100000	     16844 ns/op	 972.65 MB/s	   34915 B/op	       4 allocs/op
-BenchmarkGet/16KWithPool-4  	  100000	     14575 ns/op	1124.10 MB/s	   16483 B/op	       3 allocs/op
-BenchmarkGet/32K-4          	   50000	     27770 ns/op	1179.97 MB/s	   73827 B/op	       4 allocs/op
-BenchmarkGet/32KWithPool-4  	  100000	     24495 ns/op	1337.74 MB/s	   32867 B/op	       3 allocs/op
+BenchmarkGet/128B-4         	  316515	      3263 ns/op	  39.22 MB/s	     160 B/op	       1 allocs/op
+BenchmarkGet/256B-4         	  382551	      3204 ns/op	  79.90 MB/s	     288 B/op	       1 allocs/op
+BenchmarkGet/512B-4         	  357216	      3835 ns/op	 133.51 MB/s	     576 B/op	       1 allocs/op
+BenchmarkGet/1K-4           	  274958	      4429 ns/op	 231.20 MB/s	    1152 B/op	       1 allocs/op
+BenchmarkGet/2K-4           	  227764	      5013 ns/op	 408.55 MB/s	    2304 B/op	       1 allocs/op
+BenchmarkGet/4K-4           	  187557	      5534 ns/op	 740.15 MB/s	    4864 B/op	       1 allocs/op
+BenchmarkGet/8K-4           	  153546	      7652 ns/op	1070.56 MB/s	    9472 B/op	       1 allocs/op
+BenchmarkGet/16K-4          	  115549	     10272 ns/op	1594.95 MB/s	   18432 B/op	       1 allocs/op
+BenchmarkGet/32K-4          	   69592	     16405 ns/op	1997.39 MB/s	   40960 B/op	       1 allocs/op

-BenchmarkPut/128B-4         	  100000	     17492 ns/op	   7.32 MB/s	     441 B/op	       6 allocs/op
-BenchmarkPut/256B-4         	  100000	     17234 ns/op	  14.85 MB/s	     571 B/op	       6 allocs/op
-BenchmarkPut/512B-4         	  100000	     22837 ns/op	  22.42 MB/s	     861 B/op	       6 allocs/op
-BenchmarkPut/1K-4           	   50000	     30333 ns/op	  33.76 MB/s	    1443 B/op	       6 allocs/op
-BenchmarkPut/2K-4           	   30000	     45304 ns/op	  45.21 MB/s	    2606 B/op	       6 allocs/op
-BenchmarkPut/4K-4           	   20000	     83953 ns/op	  48.79 MB/s	    5187 B/op	       6 allocs/op
-BenchmarkPut/8K-4           	   10000	    142142 ns/op	  57.63 MB/s	    9845 B/op	       6 allocs/op
-BenchmarkPut/16K-4          	    5000	    206722 ns/op	  79.26 MB/s	   18884 B/op	       6 allocs/op
-BenchmarkPut/32K-4          	    5000	    361108 ns/op	  90.74 MB/s	   41582 B/op	       7 allocs/op
+BenchmarkPut/128BNoSync-4   	  123519	     11094 ns/op	  11.54 MB/s	      49 B/op	       2 allocs/op
+BenchmarkPut/256BNoSync-4   	   84662	     13398 ns/op	  19.11 MB/s	      50 B/op	       2 allocs/op
+BenchmarkPut/1KNoSync-4     	   46345	     24855 ns/op	  41.20 MB/s	      58 B/op	       2 allocs/op
+BenchmarkPut/2KNoSync-4     	   28820	     43817 ns/op	  46.74 MB/s	      68 B/op	       2 allocs/op
+BenchmarkPut/4KNoSync-4     	   13976	     90059 ns/op	  45.48 MB/s	      89 B/op	       2 allocs/op
+BenchmarkPut/8KNoSync-4     	    7852	    155101 ns/op	  52.82 MB/s	     130 B/op	       2 allocs/op
+BenchmarkPut/16KNoSync-4    	    4848	    238113 ns/op	  68.81 MB/s	     226 B/op	       2 allocs/op
+BenchmarkPut/32KNoSync-4    	    2564	    391483 ns/op	  83.70 MB/s	     377 B/op	       3 allocs/op

-BenchmarkScan-4             	 1000000	      1679 ns/op	     408 B/op	      16 allocs/op
+BenchmarkPut/128BSync-4     	     260	   4611273 ns/op	   0.03 MB/s	      48 B/op	       2 allocs/op
+BenchmarkPut/256BSync-4     	     265	   4665506 ns/op	   0.05 MB/s	      48 B/op	       2 allocs/op
+BenchmarkPut/1KSync-4       	     256	   4757334 ns/op	   0.22 MB/s	      48 B/op	       2 allocs/op
+BenchmarkPut/2KSync-4       	     255	   4996788 ns/op	   0.41 MB/s	      92 B/op	       2 allocs/op
+BenchmarkPut/4KSync-4       	     222	   5136481 ns/op	   0.80 MB/s	      98 B/op	       2 allocs/op
+BenchmarkPut/8KSync-4       	     223	   5530824 ns/op	   1.48 MB/s	      99 B/op	       2 allocs/op
+BenchmarkPut/16KSync-4      	     213	   5717880 ns/op	   2.87 MB/s	     202 B/op	       2 allocs/op
+BenchmarkPut/32KSync-4      	     211	   5835948 ns/op	   5.61 MB/s	     355 B/op	       3 allocs/op
+
+BenchmarkScan-4             	  568696	      2036 ns/op	     392 B/op	      33 allocs/op
 PASS
 ```

 For 128B values:

-* ~200,000 reads/sec
-* ~50,000 writes/sec
+* ~300,000 reads/sec
+* ~90,000 writes/sec
+* ~490,000 scans/sec

-The full benchmark above shows linear performance as you increase key/value sizes. Memory pooling starts to become advantageous for larger values.
-
-## Stargazers over time
-
-[![Stargazers over time](https://starcharts.herokuapp.com/prologic/bitcask.svg)](https://starcharts.herokuapp.com/prologic/bitcask)
+The full benchmark above shows linear performance as you increase key/value sizes.

 ## Support

@@ -167,7 +173,11 @@ Support the ongoing development of Bitcask!

 **Sponser**

- Become a [Sponser](https://www.patreon.com/prologic)
+- Become a [Sponsor](https://www.patreon.com/prologic)
+
+## Stargazers over time
+
+[![Stargazers over time](https://starcharts.herokuapp.com/prologic/bitcask.svg)](https://starcharts.herokuapp.com/prologic/bitcask)

 ## Contributors

@@ -175,6 +185,21 @@ Thank you to all those that have contributed to this project, battle-tested it,

 You can find an [AUTHORS](/AUTHORS) file where we keep a list of contributors to the project. If you contriibute a PR please consider adding your name there. There is also Github's own [Contributors](https://github.com/prologic/bitcask/graphs/contributors) statistics.

+[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/0)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/0)
+[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/1)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/1)
+[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/2)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/2)
+[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/3)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/3)
+[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/4)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/4)
+[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/5)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/5)
+[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/6)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/6)
+[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/7)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/7)
+
+## Related Projects
+
+- [bitraft](https://github.com/prologic/bitraft) -- A Distributed Key/Value store (_using Raft_) with a Redis compatible protocol.
+- [bitcaskfs](https://github.com/prologic/bitcaskfs) -- A FUSE filesystem for mounting a Bitcask database.
+- [bitcask-bench](https://github.com/prologic/bitcask-bench) -- A benchmarking tool comparing Bitcask and several other Go key/value libraries.
+
 ## License

 bitcask is licensed under the term of the [MIT License](https://github.com/prologic/bitcask/blob/master/LICENSE)
--- a/bitcask.go
+++ b/bitcask.go
@@ -1,19 +1,25 @@
 package bitcask

 import (
-	"encoding/json"
 	"errors"
+	"fmt"
 	"hash/crc32"
 	"io"
 	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
+	"sort"
 	"sync"

-	"github.com/gofrs/flock"
 	art "github.com/plar/go-adaptive-radix-tree"
+	"github.com/prologic/bitcask/flock"
 	"github.com/prologic/bitcask/internal"
+	"github.com/prologic/bitcask/internal/config"
+	"github.com/prologic/bitcask/internal/data"
+	"github.com/prologic/bitcask/internal/data/codec"
+	"github.com/prologic/bitcask/internal/index"
+	"github.com/prologic/bitcask/internal/metadata"
 )

 var (
@@ -24,6 +30,9 @@ var (
 	// maximum allowed key size (configured with WithMaxKeySize).
 	ErrKeyTooLarge = errors.New("error: key too large")

+	// ErrEmptyKey is the error returned for a value with an empty key.
+	ErrEmptyKey = errors.New("error: empty key")
+
 	// ErrValueTooLarge is the error returned for a value that exceeds the
 	// maximum allowed value size (configured with WithMaxValueSize).
 	ErrValueTooLarge = errors.New("error: value too large")
@@ -35,6 +44,10 @@ var (
 	// ErrDatabaseLocked is the error returned if the database is locked
 	// (typically opened by another process)
 	ErrDatabaseLocked = errors.New("error: database locked")
+
+	// ErrMergeInProgress is the error returned if merge is called when already a merge
+	// is in progress
+	ErrMergeInProgress = errors.New("error: merge already in progress")
 )

 // Bitcask is a struct that represents a on-disk LSM and WAL data structure
@@ -45,12 +58,15 @@ type Bitcask struct {

 	*flock.Flock

-	config    *config
+	config    *config.Config
 	options   []Option
 	path      string
-	curr      *internal.Datafile
-	datafiles map[int]*internal.Datafile
+	curr      data.Datafile
+	datafiles map[int]data.Datafile
 	trie      art.Tree
+	indexer   index.Indexer
+	metadata  *metadata.MetaData
+	isMerging bool
 }

 // Stats is a struct returned by Stats() on an open Bitcask instance
@@ -63,18 +79,14 @@ type Stats struct {
 // Stats returns statistics about the database including the number of
 // data files, keys and overall size on disk of the data
 func (b *Bitcask) Stats() (stats Stats, err error) {
-	var size int64
-
-	size, err = internal.DirSize(b.path)
-	if err != nil {
+	if stats.Size, err = internal.DirSize(b.path); err != nil {
 		return
 	}

-	stats.Datafiles = len(b.datafiles)
 	b.mu.RLock()
+	stats.Datafiles = len(b.datafiles)
 	stats.Keys = b.trie.Size()
 	b.mu.RUnlock()
-	stats.Size = size

 	return
 }
@@ -83,21 +95,20 @@ func (b *Bitcask) Stats() (stats Stats, err error) {
 // Close() as this is the only way to cleanup the lock held by the open
 // database.
 func (b *Bitcask) Close() error {
-	defer func() {
-		b.Flock.Unlock()
-		os.Remove(b.Flock.Path())
-	}()
+	b.mu.RLock()
+	defer b.mu.RUnlock()

-	f, err := os.OpenFile(filepath.Join(b.path, "index"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
-	if err != nil {
+	return b.close()
+}
+
+func (b *Bitcask) close() error {
+	defer b.Flock.Unlock()
+	if err := b.saveIndex(); err != nil {
 		return err
 	}
-	defer f.Close()

-	if err := internal.WriteIndex(b.trie, f); err != nil {
-		return err
-	}
-	if err := f.Sync(); err != nil {
+	b.metadata.IndexUpToDate = true
+	if err := b.saveMetadata(); err != nil {
 		return err
 	}

@@ -112,17 +123,30 @@ func (b *Bitcask) Close() error {

 // Sync flushes all buffers to disk ensuring all data is written
 func (b *Bitcask) Sync() error {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+
+	if err := b.saveMetadata(); err != nil {
+		return err
+	}
+
 	return b.curr.Sync()
 }

-// Get retrieves the value of the given key. If the key is not found or an/I/O
-// error occurs a null byte slice is returned along with the error.
+// Get fetches value for given key
 func (b *Bitcask) Get(key []byte) ([]byte, error) {
-	var df *internal.Datafile
-
 	b.mu.RLock()
+	defer b.mu.RUnlock()
+
+	return b.get(key)
+}
+
+// get retrieves the value of the given key. If the key is not found or an/I/O
+// error occurs a null byte slice is returned along with the error.
+func (b *Bitcask) get(key []byte) ([]byte, error) {
+	var df data.Datafile
+
 	value, found := b.trie.Search(key)
-	b.mu.RUnlock()
 	if !found {
 		return nil, ErrKeyNotFound
 	}
@@ -158,28 +182,38 @@ func (b *Bitcask) Has(key []byte) bool {

 // Put stores the key and value in the database.
 func (b *Bitcask) Put(key, value []byte) error {
-	if len(key) > b.config.maxKeySize {
+	if len(key) == 0 {
+		return ErrEmptyKey
+	}
+	if b.config.MaxKeySize > 0 && uint32(len(key)) > b.config.MaxKeySize {
 		return ErrKeyTooLarge
 	}
-	if len(value) > b.config.maxValueSize {
+	if b.config.MaxValueSize > 0 && uint64(len(value)) > b.config.MaxValueSize {
 		return ErrValueTooLarge
 	}

+	b.mu.Lock()
+	defer b.mu.Unlock()
 	offset, n, err := b.put(key, value)
 	if err != nil {
 		return err
 	}

-	if b.config.sync {
+	if b.config.Sync {
 		if err := b.curr.Sync(); err != nil {
 			return err
 		}
 	}

+	// in case of successful `put`, IndexUpToDate will be always be false
+	b.metadata.IndexUpToDate = false
+
+	if oldItem, found := b.trie.Search(key); found {
+		b.metadata.ReclaimableSpace += oldItem.(internal.Item).Size
+	}
+
 	item := internal.Item{FileID: b.curr.FileID(), Offset: offset, Size: n}
-	b.mu.Lock()
 	b.trie.Insert(key, item)
-	b.mu.Unlock()

 	return nil
 }
@@ -187,18 +221,40 @@ func (b *Bitcask) Put(key, value []byte) error {
 // Delete deletes the named key. If the key doesn't exist or an I/O error
 // occurs the error is returned.
 func (b *Bitcask) Delete(key []byte) error {
+	b.mu.Lock()
 	_, _, err := b.put(key, []byte{})
 	if err != nil {
+		b.mu.Unlock()
 		return err
 	}
-
-	b.mu.Lock()
+	if item, found := b.trie.Search(key); found {
+		b.metadata.ReclaimableSpace += item.(internal.Item).Size + codec.MetaInfoSize + int64(len(key))
+	}
 	b.trie.Delete(key)
 	b.mu.Unlock()

 	return nil
 }

+// DeleteAll deletes all the keys. If an I/O error occurs the error is returned.
+func (b *Bitcask) DeleteAll() (err error) {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+
+	b.trie.ForEach(func(node art.Node) bool {
+		_, _, err = b.put(node.Key(), []byte{})
+		if err != nil {
+			return false
+		}
+		item, _ := b.trie.Search(node.Key())
+		b.metadata.ReclaimableSpace += item.(internal.Item).Size + codec.MetaInfoSize + int64(len(node.Key()))
+		return true
+	})
+	b.trie = art.New()
+
+	return
+}
+
 // Scan performs a prefix scan of keys matching the given prefix and calling
 // the function `f` with the keys found. If the function returns an error
 // no further keys are processed and the first error returned.
@@ -233,12 +289,6 @@ func (b *Bitcask) Keys() chan []byte {

 		for it := b.trie.Iterator(); it.HasNext(); {
 			node, _ := it.Next()
-
-			// Skip the root node
-			if len(node.Key()) == 0 {
-				continue
-			}
-
 			ch <- node.Key()
 		}
 		close(ch)
@@ -250,26 +300,24 @@ func (b *Bitcask) Keys() chan []byte {
 // Fold iterates over all keys in the database calling the function `f` for
 // each key. If the function returns an error, no further keys are processed
 // and the error returned.
-func (b *Bitcask) Fold(f func(key []byte) error) error {
+func (b *Bitcask) Fold(f func(key []byte) error) (err error) {
 	b.mu.RLock()
 	defer b.mu.RUnlock()

 	b.trie.ForEach(func(node art.Node) bool {
-		if err := f(node.Key()); err != nil {
+		if err = f(node.Key()); err != nil {
 			return false
 		}
 		return true
 	})

-	return nil
+	return
 }

+// put inserts a new (key, value). Both key and value are valid inputs.
 func (b *Bitcask) put(key, value []byte) (int64, int64, error) {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-
 	size := b.curr.Size()
-	if size >= int64(b.config.maxDatafileSize) {
+	if size >= int64(b.config.MaxDatafileSize) {
 		err := b.curr.Close()
 		if err != nil {
 			return -1, 0, err
@@ -277,7 +325,7 @@ func (b *Bitcask) put(key, value []byte) (int64, int64, error) {

 		id := b.curr.FileID()

-		df, err := internal.NewDatafile(b.path, id, true)
+		df, err := data.NewDatafile(b.path, id, true, b.config.MaxKeySize, b.config.MaxValueSize, b.config.FileFileModeBeforeUmask)
 		if err != nil {
 			return -1, 0, err
 		}
@@ -285,104 +333,68 @@ func (b *Bitcask) put(key, value []byte) (int64, int64, error) {
 		b.datafiles[id] = df

 		id = b.curr.FileID() + 1
-		curr, err := internal.NewDatafile(b.path, id, false)
+		curr, err := data.NewDatafile(b.path, id, false, b.config.MaxKeySize, b.config.MaxValueSize, b.config.FileFileModeBeforeUmask)
 		if err != nil {
 			return -1, 0, err
 		}
 		b.curr = curr
+		err = b.saveIndex()
+		if err != nil {
+			return -1, 0, err
+		}
 	}

 	e := internal.NewEntry(key, value)
 	return b.curr.Write(e)
 }

-func (b *Bitcask) readConfig() error {
-	if internal.Exists(filepath.Join(b.path, "config.json")) {
-		data, err := ioutil.ReadFile(filepath.Join(b.path, "config.json"))
-		if err != nil {
-			return err
-		}
-
-		if err := json.Unmarshal(data, &b.config); err != nil {
-			return err
-		}
+// closeCurrentFile closes current datafile and makes it read only.
+func (b *Bitcask) closeCurrentFile() error {
+	err := b.curr.Close()
+	if err != nil {
+		return err
 	}
+	id := b.curr.FileID()
+	df, err := data.NewDatafile(b.path, id, true, b.config.MaxKeySize, b.config.MaxValueSize, b.config.FileFileModeBeforeUmask)
+	if err != nil {
+		return err
+	}
+
+	b.datafiles[id] = df
 	return nil
 }

-func (b *Bitcask) writeConfig() error {
-	data, err := json.Marshal(b.config)
+// openNewWritableFile opens new datafile for writing data
+func (b *Bitcask) openNewWritableFile() error {
+	id := b.curr.FileID() + 1
+	curr, err := data.NewDatafile(b.path, id, false, b.config.MaxKeySize, b.config.MaxValueSize, b.config.FileFileModeBeforeUmask)
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(filepath.Join(b.path, "config.json"), data, 0600)
+	b.curr = curr
+	return nil
 }

-func (b *Bitcask) reopen() error {
+func (b *Bitcask) Reopen() error {
 	b.mu.Lock()
 	defer b.mu.Unlock()

-	fns, err := internal.GetDatafiles(b.path)
+	return b.reopen()
+}
+
+// reopen reloads a bitcask object with index and datafiles
+// caller of this method should take care of locking
+func (b *Bitcask) reopen() error {
+	datafiles, lastID, err := loadDatafiles(b.path, b.config.MaxKeySize, b.config.MaxValueSize, b.config.FileFileModeBeforeUmask)
+	if err != nil {
+		return err
+	}
+	t, err := loadIndex(b.path, b.indexer, b.config.MaxKeySize, datafiles, lastID, b.metadata.IndexUpToDate)
 	if err != nil {
 		return err
 	}

-	ids, err := internal.ParseIds(fns)
-	if err != nil {
-		return err
-	}
-
-	datafiles := make(map[int]*internal.Datafile, len(ids))
-
-	for _, id := range ids {
-		df, err := internal.NewDatafile(b.path, id, true)
-		if err != nil {
-			return err
-		}
-		datafiles[id] = df
-	}
-
-	t := art.New()
-
-	if internal.Exists(path.Join(b.path, "index")) {
-		f, err := os.Open(path.Join(b.path, "index"))
-		if err != nil {
-			return err
-		}
-		defer f.Close()
-
-		if err := internal.ReadIndex(f, t); err != nil {
-			return err
-		}
-	} else {
-		for i, df := range datafiles {
-			for {
-				e, n, err := df.Read()
-				if err != nil {
-					if err == io.EOF {
-						break
-					}
-					return err
-				}
-
-				// Tombstone value  (deleted key)
-				if len(e.Value) == 0 {
-					t.Delete(e.Key)
-					continue
-				}
-
-				item := internal.Item{FileID: ids[i], Offset: e.Offset, Size: n}
-				t.Insert(e.Key, item)
-			}
-		}
-	}
-
-	var id int
-	if len(ids) > 0 {
-		id = ids[(len(ids) - 1)]
-	}
-
-	curr, err := internal.NewDatafile(b.path, id, false)
+	curr, err := data.NewDatafile(b.path, lastID, false, b.config.MaxKeySize, b.config.MaxValueSize, b.config.FileFileModeBeforeUmask)
 	if err != nil {
 		return err
 	}
@@ -398,6 +410,34 @@ func (b *Bitcask) reopen() error {
 // and deleted keys removes. Duplicate key/value pairs are also removed.
 // Call this function periodically to reclaim disk space.
 func (b *Bitcask) Merge() error {
+	b.mu.Lock()
+	if b.isMerging {
+		b.mu.Unlock()
+		return ErrMergeInProgress
+	}
+	b.isMerging = true
+	b.mu.Unlock()
+	defer func() {
+		b.isMerging = false
+	}()
+	b.mu.RLock()
+	err := b.closeCurrentFile()
+	if err != nil {
+		b.mu.RUnlock()
+		return err
+	}
+	filesToMerge := make([]int, 0, len(b.datafiles))
+	for k := range b.datafiles {
+		filesToMerge = append(filesToMerge, k)
+	}
+	err = b.openNewWritableFile()
+	if err != nil {
+		b.mu.RUnlock()
+		return err
+	}
+	b.mu.RUnlock()
+	sort.Ints(filesToMerge)
+
 	// Temporary merged database path
 	temp, err := ioutil.TempDir(b.path, "merge")
 	if err != nil {
@@ -406,7 +446,7 @@ func (b *Bitcask) Merge() error {
 	defer os.RemoveAll(temp)

 	// Create a merged database
-	mdb, err := Open(temp, b.options...)
+	mdb, err := Open(temp, withConfig(b.config))
 	if err != nil {
 		return err
 	}
@@ -415,7 +455,12 @@ func (b *Bitcask) Merge() error {
 	// Doing this automatically strips deleted keys and
 	// old key/value pairs
 	err = b.Fold(func(key []byte) error {
-		value, err := b.Get(key)
+		item, _ := b.trie.Search(key)
+		// if key was updated after start of merge operation, nothing to do
+		if item.(internal.Item).FileID > filesToMerge[len(filesToMerge)-1] {
+			return nil
+		}
+		value, err := b.get(key)
 		if err != nil {
 			return err
 		}
@@ -429,29 +474,36 @@ func (b *Bitcask) Merge() error {
 	if err != nil {
 		return err
 	}
-
-	err = mdb.Close()
-	if err != nil {
+	if err = mdb.Close(); err != nil {
+		return err
+	}
+	// no reads and writes till we reopen
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if err = b.close(); err != nil {
 		return err
 	}

-	// Close the database
-	err = b.Close()
-	if err != nil {
-		return err
-	}
-
-	// Remove all data files
+	// Remove data files
 	files, err := ioutil.ReadDir(b.path)
 	if err != nil {
 		return err
 	}
 	for _, file := range files {
-		if !file.IsDir() {
-			err := os.RemoveAll(path.Join([]string{b.path, file.Name()}...))
-			if err != nil {
-				return err
-			}
+		if file.IsDir() {
+			continue
+		}
+		ids, err := internal.ParseIds([]string{file.Name()})
+		if err != nil {
+			return err
+		}
+		// if datafile was created after start of merge, skip
+		if len(ids) > 0 && ids[0] > filesToMerge[len(filesToMerge)-1] {
+			continue
+		}
+		err = os.RemoveAll(path.Join(b.path, file.Name()))
+		if err != nil {
+			return err
 		}
 	}

@@ -469,6 +521,7 @@ func (b *Bitcask) Merge() error {
 			return err
 		}
 	}
+	b.metadata.ReclaimableSpace = 0

 	// And finally reopen the database
 	return b.reopen()
@@ -479,32 +532,45 @@ func (b *Bitcask) Merge() error {
 // configuration options as functions.
 func Open(path string, options ...Option) (*Bitcask, error) {
 	var (
-		cfg *config
-		err error
+		cfg  *config.Config
+		err  error
+		meta *metadata.MetaData
 	)

-	if err := os.MkdirAll(path, 0755); err != nil {
-		return nil, err
-	}
-
-	cfg, err = getConfig(path)
-	if err != nil {
+	configPath := filepath.Join(path, "config.json")
+	if internal.Exists(configPath) {
+		cfg, err = config.Load(configPath)
+		if err != nil {
+			return nil, err
+		}
+	} else {
 		cfg = newDefaultConfig()
 	}

-	bitcask := &Bitcask{
-		Flock:   flock.New(filepath.Join(path, "lock")),
-		config:  cfg,
-		options: options,
-		path:    path,
-	}
-
 	for _, opt := range options {
-		if err := opt(bitcask.config); err != nil {
+		if err := opt(cfg); err != nil {
 			return nil, err
 		}
 	}

+	if err := os.MkdirAll(path, cfg.DirFileModeBeforeUmask); err != nil {
+		return nil, err
+	}
+
+	meta, err = loadMetadata(path)
+	if err != nil {
+		return nil, err
+	}
+
+	bitcask := &Bitcask{
+		Flock:    flock.New(filepath.Join(path, "lock")),
+		config:   cfg,
+		options:  options,
+		path:     path,
+		indexer:  index.NewIndexer(),
+		metadata: meta,
+	}
+
 	locked, err := bitcask.Flock.TryLock()
 	if err != nil {
 		return nil, err
@@ -514,13 +580,140 @@ func Open(path string, options ...Option) (*Bitcask, error) {
 		return nil, ErrDatabaseLocked
 	}

-	if err := bitcask.writeConfig(); err != nil {
+	if err := cfg.Save(configPath); err != nil {
 		return nil, err
 	}

-	if err := bitcask.reopen(); err != nil {
+	if cfg.AutoRecovery {
+		if err := data.CheckAndRecover(path, cfg); err != nil {
+			return nil, fmt.Errorf("recovering database: %s", err)
+		}
+	}
+	if err := bitcask.Reopen(); err != nil {
 		return nil, err
 	}

 	return bitcask, nil
 }
+
+// Backup copies db directory to given path
+// it creates path if it does not exist
+func (b *Bitcask) Backup(path string) error {
+	if !internal.Exists(path) {
+		if err := os.MkdirAll(path, b.config.DirFileModeBeforeUmask); err != nil {
+			return err
+		}
+	}
+	return internal.Copy(b.path, path, []string{"lock"})
+}
+
+// saveIndex saves index currently in RAM to disk
+func (b *Bitcask) saveIndex() error {
+	tempIdx := "temp_index"
+	if err := b.indexer.Save(b.trie, filepath.Join(b.path, tempIdx)); err != nil {
+		return err
+	}
+	return os.Rename(filepath.Join(b.path, tempIdx), filepath.Join(b.path, "index"))
+}
+
+// saveMetadata saves metadata into disk
+func (b *Bitcask) saveMetadata() error {
+	return b.metadata.Save(filepath.Join(b.path, "meta.json"), b.config.DirFileModeBeforeUmask)
+}
+
+// Reclaimable returns space that can be reclaimed
+func (b *Bitcask) Reclaimable() int64 {
+	return b.metadata.ReclaimableSpace
+}
+
+func loadDatafiles(path string, maxKeySize uint32, maxValueSize uint64, fileModeBeforeUmask os.FileMode) (datafiles map[int]data.Datafile, lastID int, err error) {
+	fns, err := internal.GetDatafiles(path)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	ids, err := internal.ParseIds(fns)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	datafiles = make(map[int]data.Datafile, len(ids))
+	for _, id := range ids {
+		datafiles[id], err = data.NewDatafile(path, id, true, maxKeySize, maxValueSize, fileModeBeforeUmask)
+		if err != nil {
+			return
+		}
+
+	}
+	if len(ids) > 0 {
+		lastID = ids[len(ids)-1]
+	}
+	return
+}
+
+func getSortedDatafiles(datafiles map[int]data.Datafile) []data.Datafile {
+	out := make([]data.Datafile, len(datafiles))
+	idx := 0
+	for _, df := range datafiles {
+		out[idx] = df
+		idx++
+	}
+	sort.Slice(out, func(i, j int) bool {
+		return out[i].FileID() < out[j].FileID()
+	})
+	return out
+}
+
+func loadIndex(path string, indexer index.Indexer, maxKeySize uint32, datafiles map[int]data.Datafile, lastID int, indexUpToDate bool) (art.Tree, error) {
+	t, found, err := indexer.Load(filepath.Join(path, "index"), maxKeySize)
+	if err != nil {
+		return nil, err
+	}
+	if found && indexUpToDate {
+		return t, nil
+	}
+	if found {
+		if err := loadIndexFromDatafile(t, datafiles[lastID]); err != nil {
+			return nil, err
+		}
+		return t, nil
+	}
+	sortedDatafiles := getSortedDatafiles(datafiles)
+	for _, df := range sortedDatafiles {
+		if err := loadIndexFromDatafile(t, df); err != nil {
+			return nil, err
+		}
+	}
+	return t, nil
+}
+
+func loadIndexFromDatafile(t art.Tree, df data.Datafile) error {
+	var offset int64
+	for {
+		e, n, err := df.Read()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return err
+		}
+		// Tombstone value  (deleted key)
+		if len(e.Value) == 0 {
+			t.Delete(e.Key)
+			offset += n
+			continue
+		}
+		item := internal.Item{FileID: df.FileID(), Offset: offset, Size: n}
+		t.Insert(e.Key, item)
+		offset += n
+	}
+	return nil
+}
+
+func loadMetadata(path string) (*metadata.MetaData, error) {
+	if !internal.Exists(filepath.Join(path, "meta.json")) {
+		meta := new(metadata.MetaData)
+		return meta, nil
+	}
+	return metadata.Load(filepath.Join(path, "meta.json"))
+}
--- a/bitcask_test.go
+++ b/bitcask_test.go
--- a/cmd/bitcask/export.go
+++ b/cmd/bitcask/export.go
@@ -50,11 +50,11 @@ func init() {
 		"with-max-datafile-size", "", bitcask.DefaultMaxDatafileSize,
 		"Maximum size of each datafile",
 	)
-	exportCmd.PersistentFlags().IntP(
+	exportCmd.PersistentFlags().Uint32P(
 		"with-max-key-size", "", bitcask.DefaultMaxKeySize,
 		"Maximum size of each key",
 	)
-	exportCmd.PersistentFlags().IntP(
+	exportCmd.PersistentFlags().Uint64P(
 		"with-max-value-size", "", bitcask.DefaultMaxValueSize,
 		"Maximum size of each value",
 	)
@@ -66,11 +66,6 @@ type kvPair struct {
 }

 func export(path, output string) int {
-	var (
-		err error
-		w   io.WriteCloser
-	)
-
 	db, err := bitcask.Open(path)
 	if err != nil {
 		log.WithError(err).Error("error opening database")
@@ -78,19 +73,29 @@ func export(path, output string) int {
 	}
 	defer db.Close()

-	if output == "-" {
-		w = os.Stdout
-	} else {
-		w, err = os.OpenFile(output, os.O_WRONLY|os.O_CREATE|os.O_EXCL|os.O_TRUNC, 0755)
-		if err != nil {
+	w := os.Stdout
+	if output != "-" {
+		if w, err = os.OpenFile(output, os.O_WRONLY|os.O_CREATE|os.O_EXCL|os.O_TRUNC, 0755); err != nil {
 			log.WithError(err).
 				WithField("output", output).
 				Error("error opening output for writing")
 			return 1
 		}
+		defer w.Close()
 	}

-	err = db.Fold(func(key []byte) error {
+	if err = db.Fold(exportKey(db, w)); err != nil {
+		log.WithError(err).
+			WithField("path", path).
+			WithField("output", output).
+			Error("error exporting keys")
+		return 2
+	}
+	return 0
+}
+
+func exportKey(db *bitcask.Bitcask, w io.Writer) func(key []byte) error {
+	return func(key []byte) error {
 		value, err := db.Get(key)
 		if err != nil {
 			log.WithError(err).
@@ -129,14 +134,5 @@ func export(path, output string) int {
 		}

 		return nil
-	})
-	if err != nil {
-		log.WithError(err).
-			WithField("path", path).
-			WithField("output", output).
-			Error("error exporting keys")
-		return 2
 	}
-
-	return 0
 }
--- a/cmd/bitcask/initdb.go
+++ b/cmd/bitcask/initdb.go
@@ -30,8 +30,8 @@ var initdbCmd = &cobra.Command{
 		path := viper.GetString("path")

 		maxDatafileSize := viper.GetInt("with-max-datafile-size")
-		maxKeySize := viper.GetInt("with-max-key-size")
-		maxValueSize := viper.GetInt("with-max-value-size")
+		maxKeySize := viper.GetUint32("with-max-key-size")
+		maxValueSize := viper.GetUint64("with-max-value-size")

 		db, err := bitcask.Open(
 			path,
@@ -56,11 +56,11 @@ func init() {
 		"with-max-datafile-size", "", bitcask.DefaultMaxDatafileSize,
 		"Maximum size of each datafile",
 	)
-	initdbCmd.PersistentFlags().IntP(
+	initdbCmd.PersistentFlags().Uint32P(
 		"with-max-key-size", "", bitcask.DefaultMaxKeySize,
 		"Maximum size of each key",
 	)
-	initdbCmd.PersistentFlags().IntP(
+	initdbCmd.PersistentFlags().Uint64P(
 		"with-max-value-size", "", bitcask.DefaultMaxValueSize,
 		"Maximum size of each value",
 	)
--- a/cmd/bitcask/recover.go
+++ b/cmd/bitcask/recover.go
@@ -0,0 +1,141 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/prologic/bitcask"
+	"github.com/prologic/bitcask/internal"
+	"github.com/prologic/bitcask/internal/config"
+	"github.com/prologic/bitcask/internal/data/codec"
+	"github.com/prologic/bitcask/internal/index"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var recoveryCmd = &cobra.Command{
+	Use:     "recover",
+	Aliases: []string{"recovery"},
+	Short:   "Analyzes and recovers the index file for corruption scenarios",
+	Long: `This analyze files to detect different forms of persistence corruption in 
+persisted files. It also allows to recover the files to the latest point of integrity.
+Recovered files have the .recovered extension`,
+	Args: cobra.ExactArgs(0),
+	PreRun: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlag("dry-run", cmd.Flags().Lookup("dry-run"))
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		path := viper.GetString("path")
+		dryRun := viper.GetBool("dry-run")
+		os.Exit(recover(path, dryRun))
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(recoveryCmd)
+	recoveryCmd.Flags().BoolP("dry-run", "n", false, "Will only check files health without applying recovery if unhealthy")
+}
+
+func recover(path string, dryRun bool) int {
+	maxKeySize := bitcask.DefaultMaxKeySize
+	maxValueSize := bitcask.DefaultMaxValueSize
+	if cfg, err := config.Load(filepath.Join(path, "config.json")); err == nil {
+		maxKeySize = cfg.MaxKeySize
+		maxValueSize = cfg.MaxValueSize
+	}
+
+	if err := recoverIndex(filepath.Join(path, "index"), maxKeySize, dryRun); err != nil {
+		log.WithError(err).Info("recovering index file")
+		return 1
+	}
+
+	datafiles, err := internal.GetDatafiles(path)
+	if err != nil {
+		log.WithError(err).Info("coudn't list existing datafiles")
+		return 1
+	}
+	for _, file := range datafiles {
+		if err := recoverDatafile(file, maxKeySize, maxValueSize, dryRun); err != nil {
+			log.WithError(err).Info("recovering data file")
+			return 1
+		}
+	}
+
+	return 0
+}
+
+func recoverIndex(path string, maxKeySize uint32, dryRun bool) error {
+	t, found, err := index.NewIndexer().Load(path, maxKeySize)
+	if err != nil && !index.IsIndexCorruption(err) {
+		log.WithError(err).Info("opening the index file")
+	}
+	if !found {
+		log.Info("index file doesn't exist, will be recreated on next run.")
+		return nil
+	}
+
+	if err == nil {
+		log.Debug("index file is not corrupted")
+		return nil
+	}
+	log.Debugf("index file is corrupted: %v", err)
+
+	if dryRun {
+		log.Debug("dry-run mode, not writing to a file")
+		return nil
+	}
+
+	// Leverage that t has the partiatially read tree even on corrupted files
+	err = index.NewIndexer().Save(t, "index.recovered")
+	if err != nil {
+		return fmt.Errorf("writing the recovered index file: %w", err)
+	}
+	log.Debug("the index was recovered in the index.recovered new file")
+
+	return nil
+}
+
+func recoverDatafile(path string, maxKeySize uint32, maxValueSize uint64, dryRun bool) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("opening the datafile: %w", err)
+	}
+	defer f.Close()
+	_, file := filepath.Split(path)
+	fr, err := os.OpenFile(fmt.Sprintf("%s.recovered", file), os.O_CREATE|os.O_WRONLY, os.ModePerm)
+	if err != nil {
+		return fmt.Errorf("creating the recovered datafile: %w", err)
+	}
+	defer fr.Close()
+
+	dec := codec.NewDecoder(f, maxKeySize, maxValueSize)
+	enc := codec.NewEncoder(fr)
+	e := internal.Entry{}
+	for {
+		_, err = dec.Decode(&e)
+		if err == io.EOF {
+			break
+		}
+		if codec.IsCorruptedData(err) {
+			log.Debugf("%s is corrupted, a best-effort recovery was done", file)
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("unexpected error while reading datafile: %w", err)
+		}
+		if dryRun {
+			continue
+		}
+		if _, err := enc.Encode(e); err != nil {
+			return fmt.Errorf("writing to recovered datafile: %w", err)
+		}
+	}
+	if err := os.Remove(fr.Name()); err != nil {
+		return fmt.Errorf("can't remove temporal recovered datafile: %w", err)
+	}
+	log.Debugf("%s is not corrupted", file)
+	return nil
+}
--- a/cmd/bitcaskd/main.go
+++ b/cmd/bitcaskd/main.go
@@ -3,26 +3,22 @@ package main
 import (
 	"fmt"
 	"os"
-	"strings"

 	log "github.com/sirupsen/logrus"
 	flag "github.com/spf13/pflag"
-	"github.com/tidwall/redcon"

-	"github.com/prologic/bitcask"
 	"github.com/prologic/bitcask/internal"
 )

 var (
-	bind            string
-	debug           bool
-	version         bool
-	maxDatafileSize int
+	bind    string
+	debug   bool
+	version bool
 )

 func init() {
 	flag.Usage = func() {
-		fmt.Fprintf(os.Stderr, "Usage: %s [options] <path>\n", os.Args[0])
+		fmt.Fprintf(os.Stderr, "Usage: %s [options] <dbpath>\n", os.Args[0])
 		flag.PrintDefaults()
 	}

@@ -30,8 +26,6 @@ func init() {
 	flag.BoolVarP(&debug, "debug", "d", false, "enable debug logging")

 	flag.StringVarP(&bind, "bind", "b", ":6379", "interface and port to bind to")
-
-	flag.IntVar(&maxDatafileSize, "max-datafile-size", 1<<20, "maximum datafile size in bytes")
 }

 func main() {
@@ -55,86 +49,13 @@ func main() {

 	path := flag.Arg(0)

-	db, err := bitcask.Open(path, bitcask.WithMaxDatafileSize(maxDatafileSize))
+	server, err := newServer(bind, path)
 	if err != nil {
-		log.WithError(err).WithField("path", path).Error("error opening database")
-		os.Exit(1)
+		log.WithError(err).Error("error creating server")
+		os.Exit(2)
 	}

-	log.WithField("bind", bind).WithField("path", path).Infof("starting bitcaskd v%s", internal.FullVersion())
-
-	err = redcon.ListenAndServe(bind,
-		func(conn redcon.Conn, cmd redcon.Command) {
-			switch strings.ToLower(string(cmd.Args[0])) {
-			case "ping":
-				conn.WriteString("PONG")
-			case "quit":
-				conn.WriteString("OK")
-				conn.Close()
-			case "set":
-				if len(cmd.Args) != 3 {
-					conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
-					return
-				}
-				key := cmd.Args[1]
-				value := cmd.Args[2]
-				err = db.Put(key, value)
-				if err != nil {
-					conn.WriteString(fmt.Sprintf("ERR: %s", err))
-				} else {
-					conn.WriteString("OK")
-				}
-			case "get":
-				if len(cmd.Args) != 2 {
-					conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
-					return
-				}
-				key := cmd.Args[1]
-				value, err := db.Get(key)
-				if err != nil {
-					conn.WriteNull()
-				} else {
-					conn.WriteBulk(value)
-				}
-			case "keys":
-				conn.WriteArray(db.Len())
-				for key := range db.Keys() {
-					conn.WriteBulk([]byte(key))
-				}
-			case "exists":
-				if len(cmd.Args) != 2 {
-					conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
-					return
-				}
-				key := cmd.Args[1]
-				if db.Has(key) {
-					conn.WriteInt(1)
-				} else {
-					conn.WriteInt(0)
-				}
-			case "del":
-				if len(cmd.Args) != 2 {
-					conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
-					return
-				}
-				key := cmd.Args[1]
-				err := db.Delete(key)
-				if err != nil {
-					conn.WriteInt(0)
-				} else {
-					conn.WriteInt(1)
-				}
-			default:
-				conn.WriteError("ERR unknown command '" + string(cmd.Args[0]) + "'")
-			}
-		},
-		func(conn redcon.Conn) bool {
-			return true
-		},
-		func(conn redcon.Conn, err error) {
-		},
-	)
-	if err != nil {
-		log.WithError(err).Fatal("oops")
+	if err = server.Run(); err != nil {
+		log.Fatal(err)
 	}
 }
--- a/cmd/bitcaskd/server.go
+++ b/cmd/bitcaskd/server.go
@@ -0,0 +1,184 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/tidwall/redcon"
+
+	"github.com/prologic/bitcask"
+)
+
+type server struct {
+	bind string
+	db   *bitcask.Bitcask
+}
+
+func newServer(bind, dbpath string) (*server, error) {
+	db, err := bitcask.Open(dbpath)
+	if err != nil {
+		log.WithError(err).WithField("dbpath", dbpath).Error("error opening database")
+		return nil, err
+	}
+
+	return &server{
+		bind: bind,
+		db:   db,
+	}, nil
+}
+
+func (s *server) handleSet(cmd redcon.Command, conn redcon.Conn) {
+	if len(cmd.Args) != 3 {
+		conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
+		return
+	}
+	key := cmd.Args[1]
+	value := cmd.Args[2]
+
+	err := s.db.Lock()
+	if err != nil {
+		conn.WriteError("ERR " + fmt.Errorf("failed to lock db: %v", err).Error() + "")
+		return
+	}
+	defer s.db.Unlock()
+
+	if err := s.db.Put(key, value); err != nil {
+		conn.WriteString(fmt.Sprintf("ERR: %s", err))
+	} else {
+		conn.WriteString("OK")
+	}
+}
+
+func (s *server) handleGet(cmd redcon.Command, conn redcon.Conn) {
+	if len(cmd.Args) != 2 {
+		conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
+		return
+	}
+
+	key := cmd.Args[1]
+
+	err := s.db.Lock()
+	if err != nil {
+		conn.WriteError("ERR " + fmt.Errorf("failed to lock db: %v", err).Error() + "")
+		return
+	}
+	defer s.db.Unlock()
+
+	value, err := s.db.Get(key)
+	if err != nil {
+		conn.WriteNull()
+	} else {
+		conn.WriteBulk(value)
+	}
+}
+
+func (s *server) handleKeys(cmd redcon.Command, conn redcon.Conn) {
+	err := s.db.Lock()
+	if err != nil {
+		conn.WriteError("ERR " + fmt.Errorf("failed to lock db: %v", err).Error() + "")
+		return
+	}
+	defer s.db.Unlock()
+
+	conn.WriteArray(s.db.Len())
+	for key := range s.db.Keys() {
+		conn.WriteBulk(key)
+	}
+}
+
+func (s *server) handleExists(cmd redcon.Command, conn redcon.Conn) {
+	if len(cmd.Args) != 2 {
+		conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
+		return
+	}
+
+	key := cmd.Args[1]
+
+	err := s.db.Lock()
+	if err != nil {
+		conn.WriteError("ERR " + fmt.Errorf("failed to lock db: %v", err).Error() + "")
+		return
+	}
+	defer s.db.Unlock()
+
+	if s.db.Has(key) {
+		conn.WriteInt(1)
+	} else {
+		conn.WriteInt(0)
+	}
+}
+
+func (s *server) handleDel(cmd redcon.Command, conn redcon.Conn) {
+	if len(cmd.Args) != 2 {
+		conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
+		return
+	}
+
+	key := cmd.Args[1]
+
+	err := s.db.Lock()
+	if err != nil {
+		conn.WriteError("ERR " + fmt.Errorf("failed to lock db: %v", err).Error() + "")
+		return
+	}
+	defer s.db.Unlock()
+
+	if err := s.db.Delete(key); err != nil {
+		conn.WriteInt(0)
+	} else {
+		conn.WriteInt(1)
+	}
+}
+
+func (s *server) Shutdown() (err error) {
+	err = s.db.Close()
+	return
+}
+
+func (s *server) Run() (err error) {
+	redServer := redcon.NewServerNetwork("tcp", s.bind,
+		func(conn redcon.Conn, cmd redcon.Command) {
+			switch strings.ToLower(string(cmd.Args[0])) {
+			case "ping":
+				conn.WriteString("PONG")
+			case "quit":
+				conn.WriteString("OK")
+				conn.Close()
+			case "set":
+				s.handleSet(cmd, conn)
+			case "get":
+				s.handleGet(cmd, conn)
+			case "keys":
+				s.handleKeys(cmd, conn)
+			case "exists":
+				s.handleExists(cmd, conn)
+			case "del":
+				s.handleDel(cmd, conn)
+			default:
+				conn.WriteError("ERR unknown command '" + string(cmd.Args[0]) + "'")
+			}
+		},
+		func(conn redcon.Conn) bool {
+			return true
+		},
+		func(conn redcon.Conn, err error) {
+		},
+	)
+
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt, syscall.SIGTERM)
+		s := <-signals
+		log.Infof("Shutdown server on signal %s", s)
+		redServer.Close()
+	}()
+
+	if err := redServer.ListenAndServe(); err == nil {
+		return s.Shutdown()
+	}
+	return
+}
--- a/flock/flock.go
+++ b/flock/flock.go
@@ -0,0 +1,97 @@
+package flock
+
+import (
+	"errors"
+	"os"
+	"sync"
+)
+
+type Flock struct {
+	path string
+	m    sync.Mutex
+	fh   *os.File
+}
+
+var (
+	ErrAlreadyLocked = errors.New("Double lock: already own the lock")
+	ErrLockFailed    = errors.New("Could not acquire lock")
+	ErrLockNotHeld   = errors.New("Could not unlock, lock is not held")
+
+	ErrInodeChangedAtPath = errors.New("Inode changed at path")
+)
+
+// New returns a new instance of *Flock. The only parameter
+// it takes is the path to the desired lockfile.
+func New(path string) *Flock {
+	return &Flock{path: path}
+}
+
+// Path returns the file path linked to this lock.
+func (f *Flock) Path() string {
+	return f.path
+}
+
+// Lock will acquire the lock. This function may block indefinitely if some other process holds the lock. For a non-blocking version, see Flock.TryLock().
+func (f *Flock) Lock() error {
+	f.m.Lock()
+	defer f.m.Unlock()
+
+	if f.fh != nil {
+		return ErrAlreadyLocked
+	}
+
+	var fh *os.File
+
+	fh, err := lock_sys(f.path, false)
+	// treat "ErrInodeChangedAtPath" as "some other process holds the lock, retry locking"
+	for err == ErrInodeChangedAtPath {
+		fh, err = lock_sys(f.path, false)
+	}
+
+	if err != nil {
+		return err
+	}
+	if fh == nil {
+		return ErrLockFailed
+	}
+
+	f.fh = fh
+	return nil
+}
+
+// TryLock will try to acquire the lock, and returns immediately if the lock is already owned by another process.
+func (f *Flock) TryLock() (bool, error) {
+	f.m.Lock()
+	defer f.m.Unlock()
+
+	if f.fh != nil {
+		return false, ErrAlreadyLocked
+	}
+
+	fh, err := lock_sys(f.path, true)
+	if err != nil {
+		return false, ErrLockFailed
+	}
+
+	f.fh = fh
+	return true, nil
+}
+
+// Unlock removes the lock file from disk and releases the lock.
+// Whatever the result of `.Unlock()`, the caller must assume that it does not hold the lock anymore.
+func (f *Flock) Unlock() error {
+	f.m.Lock()
+	defer f.m.Unlock()
+
+	if f.fh == nil {
+		return ErrLockNotHeld
+	}
+
+	err1 := rm_if_match(f.fh, f.path)
+	err2 := f.fh.Close()
+
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
--- a/flock/flock_test.go
+++ b/flock/flock_test.go
@@ -0,0 +1,121 @@
+package flock
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// WARNING : this test will delete the file located at "testLockPath". Choose an adequate temporary file name.
+const testLockPath = "/tmp/bitcask_unit_test_lock" // file path to use for the lock
+
+func TestTryLock(t *testing.T) {
+	// test that basic locking functionnalities are consistent
+
+	// make sure there is no present lock when startng this test
+	os.Remove(testLockPath)
+
+	assert := assert.New(t)
+
+	lock1 := New(testLockPath)
+	lock2 := New(testLockPath)
+
+	// 1- take the first lock
+	locked1, err := lock1.TryLock()
+	assert.True(locked1)
+	assert.NoError(err)
+
+	// 2- check that the second lock cannot acquire the lock
+	locked2, err := lock2.TryLock()
+	assert.False(locked2)
+	assert.Error(err)
+
+	// 3- release the first lock
+	err = lock1.Unlock()
+	assert.NoError(err)
+
+	// 4- check that the second lock can acquire and then release the lock without error
+	locked2, err = lock2.TryLock()
+	assert.True(locked2)
+	assert.NoError(err)
+
+	err = lock2.Unlock()
+	assert.NoError(err)
+}
+
+func TestLock(t *testing.T) {
+	assert := assert.New(t)
+
+	// make sure there is no present lock when startng this test
+	os.Remove(testLockPath)
+
+	syncChan := make(chan bool)
+
+	// main goroutine: take lock on testPath
+	lock := New(testLockPath)
+
+	err := lock.Lock()
+	assert.NoError(err)
+
+	go func() {
+		// sub routine:
+		lock := New(testLockPath)
+
+		// before entering the block '.Lock()' call, signal we are about to do it
+		// see below : the main goroutine will wait for a small delay before releasing the lock
+		syncChan <- true
+		// '.Lock()' should ultimately return without error :
+		err := lock.Lock()
+		assert.NoError(err)
+
+		err = lock.Unlock()
+		assert.NoError(err)
+
+		close(syncChan)
+	}()
+
+	// wait for the "ready" signal from the sub routine,
+	<-syncChan
+
+	// after that signal wait for a small delay before releasing the lock
+	<-time.After(100 * time.Microsecond)
+	err = lock.Unlock()
+	assert.NoError(err)
+
+	// wait for the sub routine to finish
+	<-syncChan
+}
+
+func TestErrorConditions(t *testing.T) {
+	// error conditions implemented in this version :
+	// - you can't release a lock you do not hold
+	// - you can't lock twice the same lock
+
+	// -- setup
+	assert := assert.New(t)
+
+	// make sure there is no present lock when startng this test
+	os.Remove(testLockPath)
+
+	lock := New(testLockPath)
+
+	// -- run tests :
+
+	err := lock.Unlock()
+	assert.Error(err, "you can't release a lock you do not hold")
+
+	// take the lock once:
+	lock.TryLock()
+
+	locked, err := lock.TryLock()
+	assert.False(locked)
+	assert.Error(err, "you can't lock twice the same lock (using .TryLock())")
+
+	err = lock.Lock()
+	assert.Error(err, "you can't lock twice the same lock (using .Lock())")
+
+	// -- teardown
+	lock.Unlock()
+}
--- a/flock/flock_unix.go
+++ b/flock/flock_unix.go
@@ -0,0 +1,79 @@
+// +build !aix,!windows
+
+package flock
+
+import (
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+func lock_sys(path string, nonBlocking bool) (_ *os.File, err error) {
+	var fh *os.File
+
+	fh, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err != nil {
+			fh.Close()
+		}
+	}()
+
+	flag := unix.LOCK_EX
+	if nonBlocking {
+		flag |= unix.LOCK_NB
+	}
+
+	err = unix.Flock(int(fh.Fd()), flag)
+	if err != nil {
+		return nil, err
+	}
+
+	if !sameInodes(fh, path) {
+		return nil, ErrInodeChangedAtPath
+	}
+
+	return fh, nil
+}
+
+func rm_if_match(fh *os.File, path string) error {
+	// Sanity check :
+	// before running "rm", check that the file pointed at by the
+	// filehandle has the same inode as the path in the filesystem
+	//
+	// If this sanity check doesn't pass, store a "ErrInodeChangedAtPath" error,
+	// if the check passes, run os.Remove, and store the error if any.
+	//
+	// note : this sanity check is in no way atomic, but :
+	//   - as long as only cooperative processes are involved, it will work as intended
+	//   - it allows to avoid 99.9% the major pitfall case: "root user forcefully removed the lockfile"
+
+	if !sameInodes(fh, path) {
+		return ErrInodeChangedAtPath
+	}
+
+	return os.Remove(path)
+}
+
+func sameInodes(f *os.File, path string) bool {
+	// get inode from opened file f:
+	var fstat unix.Stat_t
+	err := unix.Fstat(int(f.Fd()), &fstat)
+	if err != nil {
+		return false
+	}
+	fileIno := fstat.Ino
+
+	// get inode for path on disk:
+	var dstat unix.Stat_t
+	err = unix.Stat(path, &dstat)
+	if err != nil {
+		return false
+	}
+	pathIno := dstat.Ino
+
+	return pathIno == fileIno
+}
--- a/flock/race_test.go
+++ b/flock/race_test.go
@@ -0,0 +1,236 @@
+package flock
+
+// the "nd" in "nd_test.go" stands for non-deterministic
+
+import (
+	"errors"
+	"os"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// The two tests in this file are test some concurrency scenarios :
+// 1- TestRaceLock() runs several threads racing for the same lock
+// 2- TestShatteredLock() runs racing racing threads, along with another threads which forcibly remove the file from disk
+//
+// Note that these tests are non-deterministic : the coverage produced by each test depends
+// on how the runtime chooses to schedule the concurrent goroutines.
+
+var lockerCount int64
+
+// lockAndCount tries to take a lock on "lockpath"
+// if it fails : it returns 0 and stop there
+// if it succeeds :
+//   1- it sets a defer function to release the lock in the same fashion as "func (b *Bitcask) Close()"
+//   2- it increments the shared "lockerCount" above
+//   3- it waits for a short amount of time
+//   4- it decrements "lockerCount"
+//   5- it returns the value it has seen at step 2.
+//
+// If the locking and unlocking behave as we expect them to,
+// instructions 1-5 should be in a critical section,
+// and the only possible value at step 2 should be "1".
+//
+// Returning a value > 0 indicates this function successfully acquired the lock,
+// returning a value == 0 indicates that TryLock failed.
+
+func lockAndCount(lockpath string) int64 {
+	lock := New(lockpath)
+	ok, _ := lock.TryLock()
+	if !ok {
+		return 0
+	}
+	defer func() {
+		lock.Unlock()
+	}()
+
+	x := atomic.AddInt64(&lockerCount, 1)
+	// emulate a workload :
+	<-time.After(1 * time.Microsecond)
+	atomic.AddInt64(&lockerCount, -1)
+
+	return x
+}
+
+// locker will call the lock function above in a loop, until one of the following holds :
+//  - reading from the "timeout" channel doesn't block
+//  - the number of calls to "lock()" that indicate the lock was successfully taken reaches "successfullLockCount"
+func locker(t *testing.T, id int, lockPath string, successfulLockCount int, timeout <-chan struct{}) {
+	timedOut := false
+
+	failCount := 0
+	max := int64(0)
+
+lockloop:
+	for successfulLockCount > 0 {
+		select {
+		case <-timeout:
+			timedOut = true
+			break lockloop
+		default:
+		}
+
+		x := lockAndCount(lockPath)
+
+		if x > 0 {
+			// if x indicates the lock was taken : decrement the counter
+			successfulLockCount--
+		}
+
+		if x > 1 {
+			// if x indicates an invalid value : increase the failCount and update max accordingly
+			failCount++
+			if x > max {
+				max = x
+			}
+		}
+	}
+
+	// check failure cases :
+	if timedOut {
+		t.Fail()
+		t.Logf("[runner %02d] timed out", id)
+	}
+	if failCount > 0 {
+		t.Fail()
+		t.Logf("[runner %02d] lockCounter was > 1 on %2.d occasions, max seen value was %2.d", id, failCount, max)
+	}
+}
+
+// TestRaceLock checks that no error occurs when several concurrent actors (goroutines in this case) race for the same lock.
+func TestRaceLock(t *testing.T) {
+	// test parameters, written in code :
+	//   you may want to tweak these values for testing
+
+	goroutines := 20          // number of concurrent "locker" goroutines to launch
+	successfulLockCount := 50 // how many times a "locker" will successfully take the lock before halting
+
+	// make sure there is no present lock when startng this test
+	os.Remove(testLockPath)
+
+	// timeout implemented in code
+	// (the lock acquisition depends on the behavior of the filesystem,
+	//	avoid sending CI in endless loop if something fishy happens on the test server ...)
+	// tweak this value if needed ; comment out the "close(ch)" instruction below
+	timeout := 10 * time.Second
+	ch := make(chan struct{})
+	go func() {
+		<-time.After(timeout)
+		close(ch)
+	}()
+
+	wg := &sync.WaitGroup{}
+	wg.Add(goroutines)
+
+	for i := 0; i < goroutines; i++ {
+		go func(id int) {
+			locker(t, id, testLockPath, successfulLockCount, ch)
+			wg.Done()
+		}(i)
+	}
+
+	wg.Wait()
+}
+
+func isExpectedError(err error) bool {
+	switch {
+	case err == nil:
+		return true
+	case err == ErrInodeChangedAtPath:
+		return true
+	case errors.Is(err, syscall.ENOENT):
+		return true
+
+	default:
+		return false
+	}
+}
+
+// TestShatteredLock runs concurrent goroutines on one lock, with an extra goroutine
+// which removes the lock file from disk without checking the locks
+// (e.g: a user who would run 'rm lockfile' in a loop while the program is running).
+//
+// In this scenario, errors may occur on .Unlock() ; this test checks that only errors
+// relating to the file being deleted occur.
+//
+// This test additionally logs the number of errors that occurred, grouped by error message.
+func TestShatteredLock(t *testing.T) {
+	// test parameters, written in code :
+	//   you may want to tweak these values for testing
+
+	goroutines := 4           // number of concurrent "locker" and "remover" goroutines to launch
+	successfulLockCount := 10 // how many times a "locker" will successfully take the lock before halting
+
+	// make sure there is no present lock when startng this test
+	os.Remove(testLockPath)
+	assert := assert.New(t)
+
+	wg := &sync.WaitGroup{}
+	wg.Add(goroutines)
+
+	stopChan := make(chan struct{})
+
+	errChan := make(chan error, 10)
+
+	for i := 0; i < goroutines; i++ {
+		go func(id int, count int) {
+			for count > 0 {
+				lock := New(testLockPath)
+				ok, _ := lock.TryLock()
+				if !ok {
+					continue
+				}
+
+				count--
+				err := lock.Unlock()
+				if !isExpectedError(err) {
+					assert.Fail("goroutine %d - unexpected error: %v", id, err)
+				}
+
+				if err != nil {
+					errChan <- err
+				}
+			}
+
+			wg.Done()
+		}(i, successfulLockCount)
+	}
+
+	var wgCompanion = &sync.WaitGroup{}
+	wgCompanion.Add(2)
+
+	go func() {
+		defer wgCompanion.Done()
+		for {
+			os.Remove(testLockPath)
+
+			select {
+			case <-stopChan:
+				return
+			default:
+			}
+		}
+	}()
+
+	var errs = make(map[string]int)
+	go func() {
+		for err := range errChan {
+			errs[err.Error()]++
+		}
+		wgCompanion.Done()
+	}()
+
+	wg.Wait()
+	close(stopChan)
+	close(errChan)
+	wgCompanion.Wait()
+
+	for err, count := range errs {
+		t.Logf("  seen %d times: %s", count, err)
+	}
+}
--- a/go.mod
+++ b/go.mod
@@ -1,23 +1,23 @@
 module github.com/prologic/bitcask

-go 1.12
+go 1.13

 require (
-	github.com/gofrs/flock v0.7.1
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
-	github.com/magiconair/properties v1.8.1 // indirect
-	github.com/pelletier/go-toml v1.4.0 // indirect
-	github.com/pkg/errors v0.8.1
-	github.com/plar/go-adaptive-radix-tree v1.0.1
-	github.com/sirupsen/logrus v1.4.2
+	github.com/pelletier/go-toml v1.6.0 // indirect
+	github.com/pkg/errors v0.9.1
+	github.com/plar/go-adaptive-radix-tree v1.0.4
+	github.com/sirupsen/logrus v1.7.0
 	github.com/spf13/afero v1.2.2 // indirect
-	github.com/spf13/cobra v0.0.5
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/cobra v0.0.7
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/pflag v1.0.3
-	github.com/spf13/viper v1.4.0
-	github.com/stretchr/testify v1.3.0
-	github.com/tidwall/redcon v1.0.0
-	golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56
-	golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa // indirect
-	golang.org/x/text v0.3.2 // indirect
+	github.com/spf13/pflag v1.0.5
+	github.com/spf13/viper v1.7.1
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/testify v1.6.1
+	github.com/tidwall/redcon v1.4.0
+	golang.org/x/exp v0.0.0-20200228211341-fcea875c7e85
+	golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527
+	gopkg.in/ini.v1 v1.53.0 // indirect
+	gopkg.in/yaml.v2 v2.3.0 // indirect
 )
--- a/go.sum
+++ b/go.sum
@@ -1,87 +1,153 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gofrs/flock v0.7.1 h1:DP+LD/t0njgoPBvT5MJLeliUIVQR03hiKR6vezdwHlc=
-github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg=
-github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=
+github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/plar/go-adaptive-radix-tree v1.0.1 h1:J+2qrXaKWLACw59s8SlTVYYxWjlUr/BlCsfkAzn96/0=
-github.com/plar/go-adaptive-radix-tree v1.0.1/go.mod h1:Ot8d28EII3i7Lv4PSvBlF8ejiD/CtRYDuPsySJbSaK8=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/plar/go-adaptive-radix-tree v1.0.4 h1:Ucd8R6RH2E7RW8ZtDKrsWyOD3paG2qqJO0I20WQ8oWQ=
+github.com/plar/go-adaptive-radix-tree v1.0.4/go.mod h1:Ot8d28EII3i7Lv4PSvBlF8ejiD/CtRYDuPsySJbSaK8=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -92,98 +158,198 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
-github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
-github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
+github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.7 h1:FfTH+vuMXOas8jmfb5/M7dzEYx7LpcLb7a0LPe34uOU=
+github.com/spf13/cobra v0.0.7/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
-github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
+github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/tidwall/redcon v1.0.0 h1:D4AzzJ81Afeh144fgnj5H0aSVPBBJ5RI9Rzj0zThU+E=
-github.com/tidwall/redcon v1.0.0/go.mod h1:bdYBm4rlcWpst2XMwKVzWDF9CoUxEbUmM7CQrKeOZas=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tidwall/btree v0.2.2 h1:VVo0JW/tdidNdQzNsDR4wMbL3heaxA1DGleyzQ3/niY=
+github.com/tidwall/btree v0.2.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
+github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/redcon v1.4.0 h1:y2PmDD55STRdy4S98qP/Dn+gZG+cPVvIDi9BJV2aOwA=
+github.com/tidwall/redcon v1.4.0/go.mod h1:IGzxyoKE3Ea5AWIXo/ZHP+hzY8sWXaMKr7KlFgcWSZU=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
-github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56 h1:estk1glOnSVeJ9tdEZZc5mAMDZk5lNJNyJ6DvrBkTEU=
-golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20200228211341-fcea875c7e85 h1:jqhIzSw5SQNkbu5hOGpgMHhkfXxrbsLJdkIRcX19gCY=
+golang.org/x/exp v0.0.0-20200228211341-fcea875c7e85/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa h1:KIDDMLT1O0Nr7TSxp8xM5tJcdn8tgyAONntO829og1M=
-golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.53.0 h1:c7ruDvTQi0MUTFuNpDRXLSjs7xT4TerM1icIg4uKWRg=
+gopkg.in/ini.v1 v1.53.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
--- a/internal/codec.go
+++ b/internal/codec.go
@@ -1,105 +0,0 @@
-package internal
-
-import (
-	"bufio"
-	"encoding/binary"
-	"io"
-
-	"github.com/pkg/errors"
-)
-
-const (
-	KeySize      = 4
-	ValueSize    = 8
-	checksumSize = 4
-)
-
-// NewEncoder creates a streaming Entry encoder.
-func NewEncoder(w io.Writer) *Encoder {
-	return &Encoder{w: bufio.NewWriter(w)}
-}
-
-// Encoder wraps an underlying io.Writer and allows you to stream
-// Entry encodings on it.
-type Encoder struct {
-	w *bufio.Writer
-}
-
-// Encode takes any Entry and streams it to the underlying writer.
-// Messages are framed with a key-length and value-length prefix.
-func (e *Encoder) Encode(msg Entry) (int64, error) {
-	var bufKeyValue = make([]byte, KeySize+ValueSize)
-	binary.BigEndian.PutUint32(bufKeyValue[:KeySize], uint32(len(msg.Key)))
-	binary.BigEndian.PutUint64(bufKeyValue[KeySize:KeySize+ValueSize], uint64(len(msg.Value)))
-	if _, err := e.w.Write(bufKeyValue); err != nil {
-		return 0, errors.Wrap(err, "failed writing key & value length prefix")
-	}
-
-	if _, err := e.w.Write(msg.Key); err != nil {
-		return 0, errors.Wrap(err, "failed writing key data")
-	}
-	if _, err := e.w.Write(msg.Value); err != nil {
-		return 0, errors.Wrap(err, "failed writing value data")
-	}
-
-	bufChecksumSize := bufKeyValue[:checksumSize]
-	binary.BigEndian.PutUint32(bufChecksumSize, msg.Checksum)
-	if _, err := e.w.Write(bufChecksumSize); err != nil {
-		return 0, errors.Wrap(err, "failed writing checksum data")
-	}
-
-	if err := e.w.Flush(); err != nil {
-		return 0, errors.Wrap(err, "failed flushing data")
-	}
-
-	return int64(KeySize + ValueSize + len(msg.Key) + len(msg.Value) + checksumSize), nil
-}
-
-// NewDecoder creates a streaming Entry decoder.
-func NewDecoder(r io.Reader) *Decoder {
-	return &Decoder{r: r}
-}
-
-// Decoder wraps an underlying io.Reader and allows you to stream
-// Entry decodings on it.
-type Decoder struct {
-	r io.Reader
-}
-
-func (d *Decoder) Decode(v *Entry) (int64, error) {
-	prefixBuf := make([]byte, KeySize+ValueSize)
-
-	_, err := io.ReadFull(d.r, prefixBuf)
-	if err != nil {
-		return 0, err
-	}
-
-	actualKeySize, actualValueSize := GetKeyValueSizes(prefixBuf)
-	buf := make([]byte, actualKeySize+actualValueSize+checksumSize)
-	if _, err = io.ReadFull(d.r, buf); err != nil {
-		return 0, errors.Wrap(translateError(err), "failed reading saved data")
-	}
-
-	DecodeWithoutPrefix(buf, actualKeySize, v)
-	return int64(KeySize + ValueSize + actualKeySize + actualValueSize + checksumSize), nil
-}
-
-func GetKeyValueSizes(buf []byte) (uint64, uint64) {
-	actualKeySize := binary.BigEndian.Uint32(buf[:KeySize])
-	actualValueSize := binary.BigEndian.Uint64(buf[KeySize:])
-
-	return uint64(actualKeySize), actualValueSize
-}
-
-func DecodeWithoutPrefix(buf []byte, valueOffset uint64, v *Entry) {
-	v.Key = buf[:valueOffset]
-	v.Value = buf[valueOffset : len(buf)-checksumSize]
-	v.Checksum = binary.BigEndian.Uint32(buf[len(buf)-checksumSize:])
-}
-
-func translateError(err error) error {
-	if err == io.EOF {
-		return io.ErrUnexpectedEOF
-	}
-	return err
-}
--- a/internal/codec_index.go
+++ b/internal/codec_index.go
@@ -1,110 +0,0 @@
-package internal
-
-import (
-	"encoding/binary"
-	"io"
-
-	art "github.com/plar/go-adaptive-radix-tree"
-)
-
-const (
-	Int32Size  = 4
-	Int64Size  = 8
-	FileIDSize = Int32Size
-	OffsetSize = Int64Size
-	SizeSize   = Int64Size
-)
-
-func ReadBytes(r io.Reader) ([]byte, error) {
-	s := make([]byte, Int32Size)
-	_, err := io.ReadFull(r, s)
-	if err != nil {
-		return nil, err
-	}
-	size := binary.BigEndian.Uint32(s)
-	b := make([]byte, size)
-	_, err = io.ReadFull(r, b)
-	if err != nil {
-		return nil, err
-	}
-	return b, nil
-}
-
-func WriteBytes(b []byte, w io.Writer) (int, error) {
-	s := make([]byte, Int32Size)
-	binary.BigEndian.PutUint32(s, uint32(len(b)))
-	n, err := w.Write(s)
-	if err != nil {
-		return n, err
-	}
-	m, err := w.Write(b)
-	if err != nil {
-		return (n + m), err
-	}
-	return (n + m), nil
-}
-
-func ReadItem(r io.Reader) (Item, error) {
-	buf := make([]byte, (FileIDSize + OffsetSize + SizeSize))
-	_, err := io.ReadFull(r, buf)
-	if err != nil {
-		return Item{}, err
-	}
-
-	return Item{
-		FileID: int(binary.BigEndian.Uint32(buf[:FileIDSize])),
-		Offset: int64(binary.BigEndian.Uint64(buf[FileIDSize:(FileIDSize + OffsetSize)])),
-		Size:   int64(binary.BigEndian.Uint64(buf[(FileIDSize + OffsetSize):])),
-	}, nil
-}
-
-func WriteItem(item Item, w io.Writer) (int, error) {
-	buf := make([]byte, (FileIDSize + OffsetSize + SizeSize))
-	binary.BigEndian.PutUint32(buf[:FileIDSize], uint32(item.FileID))
-	binary.BigEndian.PutUint64(buf[FileIDSize:(FileIDSize+OffsetSize)], uint64(item.Offset))
-	binary.BigEndian.PutUint64(buf[(FileIDSize+OffsetSize):], uint64(item.Size))
-	n, err := w.Write(buf)
-	if err != nil {
-		return 0, err
-	}
-	return n, nil
-}
-
-func ReadIndex(r io.Reader, t art.Tree) error {
-	for {
-		key, err := ReadBytes(r)
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			return err
-		}
-
-		item, err := ReadItem(r)
-		if err != nil {
-			return err
-		}
-
-		t.Insert(key, item)
-	}
-
-	return nil
-}
-
-func WriteIndex(t art.Tree, w io.Writer) (err error) {
-	t.ForEach(func(node art.Node) bool {
-		_, err = WriteBytes(node.Key(), w)
-		if err != nil {
-			return false
-		}
-
-		item := node.Value().(Item)
-		_, err := WriteItem(item, w)
-		if err != nil {
-			return false
-		}
-
-		return true
-	})
-	return
-}
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -0,0 +1,50 @@
+package config
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+)
+
+// Config contains the bitcask configuration parameters
+type Config struct {
+	MaxDatafileSize         int    `json:"max_datafile_size"`
+	MaxKeySize              uint32 `json:"max_key_size"`
+	MaxValueSize            uint64 `json:"max_value_size"`
+	Sync                    bool   `json:"sync"`
+	AutoRecovery            bool   `json:"autorecovery"`
+	DirFileModeBeforeUmask  os.FileMode
+	FileFileModeBeforeUmask os.FileMode
+}
+
+// Load loads a configuration from the given path
+func Load(path string) (*Config, error) {
+	var cfg Config
+
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := json.Unmarshal(data, &cfg); err != nil {
+		return nil, err
+	}
+
+	return &cfg, nil
+}
+
+// Save saves the configuration to the provided path
+func (c *Config) Save(path string) error {
+
+	data, err := json.Marshal(c)
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(path, data, c.FileFileModeBeforeUmask)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
--- a/internal/data/codec/decoder.go
+++ b/internal/data/codec/decoder.go
@@ -0,0 +1,99 @@
+package codec
+
+import (
+	"encoding/binary"
+	"io"
+
+	"github.com/pkg/errors"
+	"github.com/prologic/bitcask/internal"
+)
+
+var (
+	errInvalidKeyOrValueSize = errors.New("key/value size is invalid")
+	errCantDecodeOnNilEntry  = errors.New("can't decode on nil entry")
+	errTruncatedData         = errors.New("data is truncated")
+)
+
+// NewDecoder creates a streaming Entry decoder.
+func NewDecoder(r io.Reader, maxKeySize uint32, maxValueSize uint64) *Decoder {
+	return &Decoder{
+		r:            r,
+		maxKeySize:   maxKeySize,
+		maxValueSize: maxValueSize,
+	}
+}
+
+// Decoder wraps an underlying io.Reader and allows you to stream
+// Entry decodings on it.
+type Decoder struct {
+	r            io.Reader
+	maxKeySize   uint32
+	maxValueSize uint64
+}
+
+// Decode decodes the next Entry from the current stream
+func (d *Decoder) Decode(v *internal.Entry) (int64, error) {
+	if v == nil {
+		return 0, errCantDecodeOnNilEntry
+	}
+
+	prefixBuf := make([]byte, keySize+valueSize)
+
+	_, err := io.ReadFull(d.r, prefixBuf)
+	if err != nil {
+		return 0, err
+	}
+
+	actualKeySize, actualValueSize, err := getKeyValueSizes(prefixBuf, d.maxKeySize, d.maxValueSize)
+	if err != nil {
+		return 0, err
+	}
+
+	buf := make([]byte, uint64(actualKeySize)+actualValueSize+checksumSize)
+	if _, err = io.ReadFull(d.r, buf); err != nil {
+		return 0, errTruncatedData
+	}
+
+	decodeWithoutPrefix(buf, actualKeySize, v)
+	return int64(keySize + valueSize + uint64(actualKeySize) + actualValueSize + checksumSize), nil
+}
+
+// DecodeEntry decodes a serialized entry
+func DecodeEntry(b []byte, e *internal.Entry, maxKeySize uint32, maxValueSize uint64) error {
+	valueOffset, _, err := getKeyValueSizes(b, maxKeySize, maxValueSize)
+	if err != nil {
+		return errors.Wrap(err, "key/value sizes are invalid")
+	}
+
+	decodeWithoutPrefix(b[keySize+valueSize:], valueOffset, e)
+
+	return nil
+}
+
+func getKeyValueSizes(buf []byte, maxKeySize uint32, maxValueSize uint64) (uint32, uint64, error) {
+	actualKeySize := binary.BigEndian.Uint32(buf[:keySize])
+	actualValueSize := binary.BigEndian.Uint64(buf[keySize:])
+
+	if (maxKeySize > 0 && actualKeySize > maxKeySize) || (maxValueSize > 0 && actualValueSize > maxValueSize) || actualKeySize == 0 {
+
+		return 0, 0, errInvalidKeyOrValueSize
+	}
+
+	return actualKeySize, actualValueSize, nil
+}
+
+func decodeWithoutPrefix(buf []byte, valueOffset uint32, v *internal.Entry) {
+	v.Key = buf[:valueOffset]
+	v.Value = buf[valueOffset : len(buf)-checksumSize]
+	v.Checksum = binary.BigEndian.Uint32(buf[len(buf)-checksumSize:])
+}
+
+// IsCorruptedData indicates if the error correspondes to possible data corruption
+func IsCorruptedData(err error) bool {
+	switch err {
+	case errCantDecodeOnNilEntry, errInvalidKeyOrValueSize, errTruncatedData:
+		return true
+	default:
+		return false
+	}
+}
--- a/internal/data/codec/decoder_test.go
+++ b/internal/data/codec/decoder_test.go
@@ -0,0 +1,109 @@
+package codec
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+	"testing"
+
+	"github.com/prologic/bitcask/internal"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDecodeOnNilEntry(t *testing.T) {
+	t.Parallel()
+	assert := assert.New(t)
+	decoder := NewDecoder(&bytes.Buffer{}, 1, 1)
+
+	_, err := decoder.Decode(nil)
+	if assert.Error(err) {
+		assert.Equal(errCantDecodeOnNilEntry, err)
+	}
+}
+
+func TestShortPrefix(t *testing.T) {
+	t.Parallel()
+	assert := assert.New(t)
+	maxKeySize, maxValueSize := uint32(10), uint64(20)
+	prefix := make([]byte, keySize+valueSize)
+	binary.BigEndian.PutUint32(prefix, 1)
+	binary.BigEndian.PutUint64(prefix[keySize:], 1)
+
+	truncBytesCount := 2
+	buf := bytes.NewBuffer(prefix[:keySize+valueSize-truncBytesCount])
+	decoder := NewDecoder(buf, maxKeySize, maxValueSize)
+	_, err := decoder.Decode(&internal.Entry{})
+	if assert.Error(err) {
+		assert.Equal(io.ErrUnexpectedEOF, err)
+	}
+}
+
+func TestInvalidValueKeySizes(t *testing.T) {
+	assert := assert.New(t)
+	maxKeySize, maxValueSize := uint32(10), uint64(20)
+
+	tests := []struct {
+		keySize   uint32
+		valueSize uint64
+		name      string
+	}{
+		{keySize: 0, valueSize: 5, name: "zero key size"}, //zero value size is correct for tombstones
+		{keySize: 11, valueSize: 5, name: "key size overflow"},
+		{keySize: 5, valueSize: 21, name: "value size overflow"},
+		{keySize: 11, valueSize: 21, name: "key and value size overflow"},
+	}
+
+	for i := range tests {
+		i := i
+		t.Run(tests[i].name, func(t *testing.T) {
+			t.Parallel()
+			prefix := make([]byte, keySize+valueSize)
+			binary.BigEndian.PutUint32(prefix, tests[i].keySize)
+			binary.BigEndian.PutUint64(prefix[keySize:], tests[i].valueSize)
+
+			buf := bytes.NewBuffer(prefix)
+			decoder := NewDecoder(buf, maxKeySize, maxValueSize)
+			_, err := decoder.Decode(&internal.Entry{})
+			if assert.Error(err) {
+				assert.Equal(errInvalidKeyOrValueSize, err)
+			}
+		})
+	}
+}
+
+func TestTruncatedData(t *testing.T) {
+	assert := assert.New(t)
+	maxKeySize, maxValueSize := uint32(10), uint64(20)
+
+	key := []byte("foo")
+	value := []byte("bar")
+	data := make([]byte, keySize+valueSize+len(key)+len(value)+checksumSize)
+
+	binary.BigEndian.PutUint32(data, uint32(len(key)))
+	binary.BigEndian.PutUint64(data[keySize:], uint64(len(value)))
+	copy(data[keySize+valueSize:], key)
+	copy(data[keySize+valueSize+len(key):], value)
+	copy(data[keySize+valueSize+len(key)+len(value):], bytes.Repeat([]byte("0"), checksumSize))
+
+	tests := []struct {
+		data []byte
+		name string
+	}{
+		{data: data[:keySize+valueSize+len(key)-1], name: "truncated key"},
+		{data: data[:keySize+valueSize+len(key)+len(value)-1], name: "truncated value"},
+		{data: data[:keySize+valueSize+len(key)+len(value)+checksumSize-1], name: "truncated checksum"},
+	}
+
+	for i := range tests {
+		i := i
+		t.Run(tests[i].name, func(t *testing.T) {
+			t.Parallel()
+			buf := bytes.NewBuffer(tests[i].data)
+			decoder := NewDecoder(buf, maxKeySize, maxValueSize)
+			_, err := decoder.Decode(&internal.Entry{})
+			if assert.Error(err) {
+				assert.Equal(errTruncatedData, err)
+			}
+		})
+	}
+}
--- a/internal/data/codec/encoder.go
+++ b/internal/data/codec/encoder.go
@@ -0,0 +1,58 @@
+package codec
+
+import (
+	"bufio"
+	"encoding/binary"
+	"io"
+
+	"github.com/pkg/errors"
+	"github.com/prologic/bitcask/internal"
+)
+
+const (
+	keySize      = 4
+	valueSize    = 8
+	checksumSize = 4
+	MetaInfoSize = keySize + valueSize + checksumSize
+)
+
+// NewEncoder creates a streaming Entry encoder.
+func NewEncoder(w io.Writer) *Encoder {
+	return &Encoder{w: bufio.NewWriter(w)}
+}
+
+// Encoder wraps an underlying io.Writer and allows you to stream
+// Entry encodings on it.
+type Encoder struct {
+	w *bufio.Writer
+}
+
+// Encode takes any Entry and streams it to the underlying writer.
+// Messages are framed with a key-length and value-length prefix.
+func (e *Encoder) Encode(msg internal.Entry) (int64, error) {
+	var bufKeyValue = make([]byte, keySize+valueSize)
+	binary.BigEndian.PutUint32(bufKeyValue[:keySize], uint32(len(msg.Key)))
+	binary.BigEndian.PutUint64(bufKeyValue[keySize:keySize+valueSize], uint64(len(msg.Value)))
+	if _, err := e.w.Write(bufKeyValue); err != nil {
+		return 0, errors.Wrap(err, "failed writing key & value length prefix")
+	}
+
+	if _, err := e.w.Write(msg.Key); err != nil {
+		return 0, errors.Wrap(err, "failed writing key data")
+	}
+	if _, err := e.w.Write(msg.Value); err != nil {
+		return 0, errors.Wrap(err, "failed writing value data")
+	}
+
+	bufChecksumSize := bufKeyValue[:checksumSize]
+	binary.BigEndian.PutUint32(bufChecksumSize, msg.Checksum)
+	if _, err := e.w.Write(bufChecksumSize); err != nil {
+		return 0, errors.Wrap(err, "failed writing checksum data")
+	}
+
+	if err := e.w.Flush(); err != nil {
+		return 0, errors.Wrap(err, "failed flushing data")
+	}
+
+	return int64(keySize + valueSize + len(msg.Key) + len(msg.Value) + checksumSize), nil
+}
--- a/internal/data/codec/encoder_test.go
+++ b/internal/data/codec/encoder_test.go
@@ -0,0 +1,29 @@
+package codec
+
+import (
+	"bytes"
+	"encoding/hex"
+	"testing"
+
+	"github.com/prologic/bitcask/internal"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEncode(t *testing.T) {
+	t.Parallel()
+	assert := assert.New(t)
+
+	var buf bytes.Buffer
+	encoder := NewEncoder(&buf)
+	_, err := encoder.Encode(internal.Entry{
+		Key:      []byte("mykey"),
+		Value:    []byte("myvalue"),
+		Checksum: 414141,
+		Offset:   424242,
+	})
+
+	expectedHex := "0000000500000000000000076d796b65796d7976616c7565000651bd"
+	if assert.NoError(err) {
+		assert.Equal(expectedHex, hex.EncodeToString(buf.Bytes()))
+	}
+}
--- a/internal/data/datafile.go
+++ b/internal/data/datafile.go
@@ -0,0 +1,193 @@
+package data
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/prologic/bitcask/internal"
+	"github.com/prologic/bitcask/internal/data/codec"
+	"golang.org/x/exp/mmap"
+)
+
+const (
+	defaultDatafileFilename = "%09d.data"
+)
+
+var (
+	errReadonly  = errors.New("error: read only datafile")
+	errReadError = errors.New("error: read error")
+)
+
+// Datafile is an interface  that represents a readable and writeable datafile
+type Datafile interface {
+	FileID() int
+	Name() string
+	Close() error
+	Sync() error
+	Size() int64
+	Read() (internal.Entry, int64, error)
+	ReadAt(index, size int64) (internal.Entry, error)
+	Write(internal.Entry) (int64, int64, error)
+}
+
+type datafile struct {
+	sync.RWMutex
+
+	id           int
+	r            *os.File
+	ra           *mmap.ReaderAt
+	w            *os.File
+	offset       int64
+	dec          *codec.Decoder
+	enc          *codec.Encoder
+	maxKeySize   uint32
+	maxValueSize uint64
+}
+
+// NewDatafile opens an existing datafile
+func NewDatafile(path string, id int, readonly bool, maxKeySize uint32, maxValueSize uint64, fileMode os.FileMode) (Datafile, error) {
+	var (
+		r   *os.File
+		ra  *mmap.ReaderAt
+		w   *os.File
+		err error
+	)
+
+	fn := filepath.Join(path, fmt.Sprintf(defaultDatafileFilename, id))
+
+	if !readonly {
+		w, err = os.OpenFile(fn, os.O_WRONLY|os.O_APPEND|os.O_CREATE, fileMode)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	r, err = os.Open(fn)
+	if err != nil {
+		return nil, err
+	}
+	stat, err := r.Stat()
+	if err != nil {
+		return nil, errors.Wrap(err, "error calling Stat()")
+	}
+
+	ra, err = mmap.Open(fn)
+	if err != nil {
+		return nil, err
+	}
+
+	offset := stat.Size()
+
+	dec := codec.NewDecoder(r, maxKeySize, maxValueSize)
+	enc := codec.NewEncoder(w)
+
+	return &datafile{
+		id:           id,
+		r:            r,
+		ra:           ra,
+		w:            w,
+		offset:       offset,
+		dec:          dec,
+		enc:          enc,
+		maxKeySize:   maxKeySize,
+		maxValueSize: maxValueSize,
+	}, nil
+}
+
+func (df *datafile) FileID() int {
+	return df.id
+}
+
+func (df *datafile) Name() string {
+	return df.r.Name()
+}
+
+func (df *datafile) Close() error {
+	defer func() {
+		df.ra.Close()
+		df.r.Close()
+	}()
+
+	// Readonly datafile -- Nothing further to close on the write side
+	if df.w == nil {
+		return nil
+	}
+
+	err := df.Sync()
+	if err != nil {
+		return err
+	}
+	return df.w.Close()
+}
+
+func (df *datafile) Sync() error {
+	if df.w == nil {
+		return nil
+	}
+	return df.w.Sync()
+}
+
+func (df *datafile) Size() int64 {
+	df.RLock()
+	defer df.RUnlock()
+	return df.offset
+}
+
+// Read reads the next entry from the datafile
+func (df *datafile) Read() (e internal.Entry, n int64, err error) {
+	df.Lock()
+	defer df.Unlock()
+
+	n, err = df.dec.Decode(&e)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+// ReadAt the entry located at index offset with expected serialized size
+func (df *datafile) ReadAt(index, size int64) (e internal.Entry, err error) {
+	var n int
+
+	b := make([]byte, size)
+
+	if df.w == nil {
+		n, err = df.ra.ReadAt(b, index)
+	} else {
+		n, err = df.r.ReadAt(b, index)
+	}
+	if err != nil {
+		return
+	}
+	if int64(n) != size {
+		err = errReadError
+		return
+	}
+
+	codec.DecodeEntry(b, &e, df.maxKeySize, df.maxValueSize)
+
+	return
+}
+
+func (df *datafile) Write(e internal.Entry) (int64, int64, error) {
+	if df.w == nil {
+		return -1, 0, errReadonly
+	}
+
+	df.Lock()
+	defer df.Unlock()
+
+	e.Offset = df.offset
+
+	n, err := df.enc.Encode(e)
+	if err != nil {
+		return -1, 0, err
+	}
+	df.offset += n
+
+	return e.Offset, n, nil
+}
--- a/internal/data/recover.go
+++ b/internal/data/recover.go
@@ -0,0 +1,95 @@
+package data
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/prologic/bitcask/internal"
+	"github.com/prologic/bitcask/internal/config"
+	"github.com/prologic/bitcask/internal/data/codec"
+)
+
+// CheckAndRecover checks and recovers the last datafile.
+// If the datafile isn't corrupted, this is a noop. If it is,
+// the longest non-corrupted prefix will be kept and the rest
+// will be *deleted*. Also, the index file is also *deleted* which
+// will be automatically recreated on next startup.
+func CheckAndRecover(path string, cfg *config.Config) error {
+	dfs, err := internal.GetDatafiles(path)
+	if err != nil {
+		return fmt.Errorf("scanning datafiles: %s", err)
+	}
+	if len(dfs) == 0 {
+		return nil
+	}
+	f := dfs[len(dfs)-1]
+	recovered, err := recoverDatafile(f, cfg)
+	if err != nil {
+		return fmt.Errorf("recovering data file")
+	}
+	if recovered {
+		if err := os.Remove(filepath.Join(path, "index")); err != nil {
+			return fmt.Errorf("error deleting the index on recovery: %s", err)
+		}
+	}
+	return nil
+}
+
+func recoverDatafile(path string, cfg *config.Config) (recovered bool, err error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return false, fmt.Errorf("opening the datafile: %s", err)
+	}
+	defer func() {
+		closeErr := f.Close()
+		if err == nil {
+			err = closeErr
+		}
+	}()
+	_, file := filepath.Split(path)
+	rPath := fmt.Sprintf("%s.recovered", file)
+	fr, err := os.OpenFile(rPath, os.O_CREATE|os.O_WRONLY, os.ModePerm)
+	if err != nil {
+		return false, fmt.Errorf("creating the recovered datafile: %w", err)
+	}
+	defer func() {
+		closeErr := fr.Close()
+		if err == nil {
+			err = closeErr
+		}
+	}()
+
+	dec := codec.NewDecoder(f, cfg.MaxKeySize, cfg.MaxValueSize)
+	enc := codec.NewEncoder(fr)
+	e := internal.Entry{}
+
+	corrupted := false
+	for !corrupted {
+		_, err = dec.Decode(&e)
+		if err == io.EOF {
+			break
+		}
+		if codec.IsCorruptedData(err) {
+			corrupted = true
+			continue
+		}
+		if err != nil {
+			return false, fmt.Errorf("unexpected error while reading datafile: %w", err)
+		}
+		if _, err := enc.Encode(e); err != nil {
+			return false, fmt.Errorf("writing to recovered datafile: %w", err)
+		}
+	}
+	if !corrupted {
+		if err := os.Remove(fr.Name()); err != nil {
+			return false, fmt.Errorf("can't remove temporal recovered datafile: %w", err)
+		}
+		return false, nil
+	}
+	if err := os.Rename(rPath, path); err != nil {
+		return false, fmt.Errorf("removing corrupted file: %s", err)
+	}
+	return true, nil
+}
--- a/internal/datafile.go
+++ b/internal/datafile.go
@@ -1,175 +0,0 @@
-package internal
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"sync"
-
-	"github.com/pkg/errors"
-	"golang.org/x/exp/mmap"
-)
-
-const (
-	DefaultDatafileFilename = "%09d.data"
-)
-
-var (
-	ErrReadonly  = errors.New("error: read only datafile")
-	ErrReadError = errors.New("error: read error")
-
-	mxMemPool sync.RWMutex
-)
-
-type Datafile struct {
-	sync.RWMutex
-
-	id     int
-	r      *os.File
-	ra     *mmap.ReaderAt
-	w      *os.File
-	offset int64
-	dec    *Decoder
-	enc    *Encoder
-}
-
-func NewDatafile(path string, id int, readonly bool) (*Datafile, error) {
-	var (
-		r   *os.File
-		ra  *mmap.ReaderAt
-		w   *os.File
-		err error
-	)
-
-	fn := filepath.Join(path, fmt.Sprintf(DefaultDatafileFilename, id))
-
-	if !readonly {
-		w, err = os.OpenFile(fn, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0640)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	r, err = os.Open(fn)
-	if err != nil {
-		return nil, err
-	}
-	stat, err := r.Stat()
-	if err != nil {
-		return nil, errors.Wrap(err, "error calling Stat()")
-	}
-
-	ra, err = mmap.Open(fn)
-	if err != nil {
-		return nil, err
-	}
-
-	offset := stat.Size()
-
-	dec := NewDecoder(r)
-	enc := NewEncoder(w)
-
-	return &Datafile{
-		id:     id,
-		r:      r,
-		ra:     ra,
-		w:      w,
-		offset: offset,
-		dec:    dec,
-		enc:    enc,
-	}, nil
-}
-
-func (df *Datafile) FileID() int {
-	return df.id
-}
-
-func (df *Datafile) Name() string {
-	return df.r.Name()
-}
-
-func (df *Datafile) Close() error {
-	defer func() {
-		df.ra.Close()
-		df.r.Close()
-	}()
-
-	// Readonly Datafile -- Nothing further to close on the write side
-	if df.w == nil {
-		return nil
-	}
-
-	err := df.Sync()
-	if err != nil {
-		return err
-	}
-	return df.w.Close()
-}
-
-func (df *Datafile) Sync() error {
-	if df.w == nil {
-		return nil
-	}
-	return df.w.Sync()
-}
-
-func (df *Datafile) Size() int64 {
-	df.RLock()
-	defer df.RUnlock()
-	return df.offset
-}
-
-func (df *Datafile) Read() (e Entry, n int64, err error) {
-	df.Lock()
-	defer df.Unlock()
-
-	n, err = df.dec.Decode(&e)
-	if err != nil {
-		return
-	}
-
-	return
-}
-
-func (df *Datafile) ReadAt(index, size int64) (e Entry, err error) {
-	var n int
-
-	b := make([]byte, size)
-
-	if df.w == nil {
-		n, err = df.ra.ReadAt(b, index)
-	} else {
-		n, err = df.r.ReadAt(b, index)
-	}
-	if err != nil {
-		return
-	}
-	if int64(n) != size {
-		err = ErrReadError
-		return
-	}
-
-	valueOffset, _ := GetKeyValueSizes(b)
-	DecodeWithoutPrefix(b[KeySize+ValueSize:], valueOffset, &e)
-
-	return
-}
-
-func (df *Datafile) Write(e Entry) (int64, int64, error) {
-	if df.w == nil {
-		return -1, 0, ErrReadonly
-	}
-
-	df.Lock()
-	defer df.Unlock()
-
-	e.Offset = df.offset
-
-	n, err := df.enc.Encode(e)
-	if err != nil {
-		return -1, 0, err
-	}
-	df.offset += n
-
-	return e.Offset, n, nil
-}
--- a/internal/entry.go
+++ b/internal/entry.go
@@ -12,6 +12,7 @@ type Entry struct {
 	Value    []byte
 }

+// NewEntry creates a new `Entry` with the given `key` and `value`
 func NewEntry(key, value []byte) Entry {
 	checksum := crc32.ChecksumIEEE(value)

--- a/internal/index/codec_index.go
+++ b/internal/index/codec_index.go
@@ -0,0 +1,134 @@
+package index
+
+import (
+	"encoding/binary"
+	"io"
+
+	"github.com/pkg/errors"
+	art "github.com/plar/go-adaptive-radix-tree"
+	"github.com/prologic/bitcask/internal"
+)
+
+var (
+	errTruncatedKeySize = errors.New("key size is truncated")
+	errTruncatedKeyData = errors.New("key data is truncated")
+	errTruncatedData    = errors.New("data is truncated")
+	errKeySizeTooLarge  = errors.New("key size too large")
+)
+
+const (
+	int32Size  = 4
+	int64Size  = 8
+	fileIDSize = int32Size
+	offsetSize = int64Size
+	sizeSize   = int64Size
+)
+
+func readKeyBytes(r io.Reader, maxKeySize uint32) ([]byte, error) {
+	s := make([]byte, int32Size)
+	_, err := io.ReadFull(r, s)
+	if err != nil {
+		if err == io.EOF {
+			return nil, err
+		}
+		return nil, errors.Wrap(errTruncatedKeySize, err.Error())
+	}
+	size := binary.BigEndian.Uint32(s)
+	if maxKeySize > 0 && size > uint32(maxKeySize) {
+		return nil, errKeySizeTooLarge
+	}
+
+	b := make([]byte, size)
+	_, err = io.ReadFull(r, b)
+	if err != nil {
+		return nil, errors.Wrap(errTruncatedKeyData, err.Error())
+	}
+	return b, nil
+}
+
+func writeBytes(b []byte, w io.Writer) error {
+	s := make([]byte, int32Size)
+	binary.BigEndian.PutUint32(s, uint32(len(b)))
+	_, err := w.Write(s)
+	if err != nil {
+		return err
+	}
+	_, err = w.Write(b)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func readItem(r io.Reader) (internal.Item, error) {
+	buf := make([]byte, (fileIDSize + offsetSize + sizeSize))
+	_, err := io.ReadFull(r, buf)
+	if err != nil {
+		return internal.Item{}, errors.Wrap(errTruncatedData, err.Error())
+	}
+
+	return internal.Item{
+		FileID: int(binary.BigEndian.Uint32(buf[:fileIDSize])),
+		Offset: int64(binary.BigEndian.Uint64(buf[fileIDSize:(fileIDSize + offsetSize)])),
+		Size:   int64(binary.BigEndian.Uint64(buf[(fileIDSize + offsetSize):])),
+	}, nil
+}
+
+func writeItem(item internal.Item, w io.Writer) error {
+	buf := make([]byte, (fileIDSize + offsetSize + sizeSize))
+	binary.BigEndian.PutUint32(buf[:fileIDSize], uint32(item.FileID))
+	binary.BigEndian.PutUint64(buf[fileIDSize:(fileIDSize+offsetSize)], uint64(item.Offset))
+	binary.BigEndian.PutUint64(buf[(fileIDSize+offsetSize):], uint64(item.Size))
+	_, err := w.Write(buf)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// ReadIndex reads a persisted from a io.Reader into a Tree
+func readIndex(r io.Reader, t art.Tree, maxKeySize uint32) error {
+	for {
+		key, err := readKeyBytes(r, maxKeySize)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return err
+		}
+
+		item, err := readItem(r)
+		if err != nil {
+			return err
+		}
+
+		t.Insert(key, item)
+	}
+
+	return nil
+}
+
+func writeIndex(t art.Tree, w io.Writer) (err error) {
+	t.ForEach(func(node art.Node) bool {
+		err = writeBytes(node.Key(), w)
+		if err != nil {
+			return false
+		}
+
+		item := node.Value().(internal.Item)
+		err := writeItem(item, w)
+		return err == nil
+	})
+	return
+}
+
+// IsIndexCorruption returns a boolean indicating whether the error
+// is known to report a corruption data issue
+func IsIndexCorruption(err error) bool {
+	cause := errors.Cause(err)
+	switch cause {
+	case errKeySizeTooLarge, errTruncatedData, errTruncatedKeyData, errTruncatedKeySize:
+		return true
+	}
+	return false
+}
--- a/internal/index/codec_index_test.go
+++ b/internal/index/codec_index_test.go
@@ -0,0 +1,126 @@
+package index
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/binary"
+	"testing"
+
+	"github.com/pkg/errors"
+	art "github.com/plar/go-adaptive-radix-tree"
+	"github.com/prologic/bitcask/internal"
+)
+
+const (
+	base64SampleTree = "AAAABGFiY2QAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAARhYmNlAAAAAQAAAAAAAAABAAAAAAAAAAEAAAAEYWJjZgAAAAIAAAAAAAAAAgAAAAAAAAACAAAABGFiZ2QAAAADAAAAAAAAAAMAAAAAAAAAAw=="
+)
+
+func TestWriteIndex(t *testing.T) {
+	at, expectedSerializedSize := getSampleTree()
+
+	var b bytes.Buffer
+	err := writeIndex(at, &b)
+	if err != nil {
+		t.Fatalf("writing index failed: %v", err)
+	}
+	if b.Len() != expectedSerializedSize {
+		t.Fatalf("incorrect size of serialied index: expected %d, got: %d", expectedSerializedSize, b.Len())
+	}
+	sampleTreeBytes, _ := base64.StdEncoding.DecodeString(base64SampleTree)
+	if !bytes.Equal(b.Bytes(), sampleTreeBytes) {
+		t.Fatalf("unexpected serialization of the tree")
+	}
+}
+
+func TestReadIndex(t *testing.T) {
+	sampleTreeBytes, _ := base64.StdEncoding.DecodeString(base64SampleTree)
+	b := bytes.NewBuffer(sampleTreeBytes)
+
+	at := art.New()
+	err := readIndex(b, at, 1024)
+	if err != nil {
+		t.Fatalf("error while deserializing correct sample tree: %v", err)
+	}
+
+	atsample, _ := getSampleTree()
+	if atsample.Size() != at.Size() {
+		t.Fatalf("trees aren't the same size, expected %v, got %v", atsample.Size(), at.Size())
+	}
+	atsample.ForEach(func(node art.Node) bool {
+		_, found := at.Search(node.Key())
+		if !found {
+			t.Fatalf("expected node wasn't found: %s", node.Key())
+		}
+		return true
+	})
+}
+
+func TestReadCorruptedData(t *testing.T) {
+	sampleBytes, _ := base64.StdEncoding.DecodeString(base64SampleTree)
+
+	t.Run("truncated", func(t *testing.T) {
+		table := []struct {
+			name string
+			err  error
+			data []byte
+		}{
+			{name: "key-size-first-item", err: errTruncatedKeySize, data: sampleBytes[:2]},
+			{name: "key-data-second-item", err: errTruncatedKeyData, data: sampleBytes[:6]},
+			{name: "key-size-second-item", err: errTruncatedKeySize, data: sampleBytes[:(int32Size+4+fileIDSize+offsetSize+sizeSize)+2]},
+			{name: "key-data-second-item", err: errTruncatedKeyData, data: sampleBytes[:(int32Size+4+fileIDSize+offsetSize+sizeSize)+6]},
+			{name: "data", err: errTruncatedData, data: sampleBytes[:int32Size+4+(fileIDSize+offsetSize+sizeSize-3)]},
+		}
+
+		for i := range table {
+			t.Run(table[i].name, func(t *testing.T) {
+				bf := bytes.NewBuffer(table[i].data)
+
+				if err := readIndex(bf, art.New(), 1024); !IsIndexCorruption(err) || errors.Cause(err) != table[i].err {
+					t.Fatalf("expected %v, got %v", table[i].err, err)
+				}
+			})
+		}
+	})
+
+	t.Run("overflow", func(t *testing.T) {
+		overflowKeySize := make([]byte, len(sampleBytes))
+		copy(overflowKeySize, sampleBytes)
+		binary.BigEndian.PutUint32(overflowKeySize, 1025)
+
+		overflowDataSize := make([]byte, len(sampleBytes))
+		copy(overflowDataSize, sampleBytes)
+		binary.BigEndian.PutUint32(overflowDataSize[int32Size+4+fileIDSize+offsetSize:], 1025)
+
+		table := []struct {
+			name       string
+			err        error
+			maxKeySize uint32
+			data       []byte
+		}{
+			{name: "key-data-overflow", err: errKeySizeTooLarge, maxKeySize: 1024, data: overflowKeySize},
+		}
+
+		for i := range table {
+			t.Run(table[i].name, func(t *testing.T) {
+				bf := bytes.NewBuffer(table[i].data)
+
+				if err := readIndex(bf, art.New(), table[i].maxKeySize); !IsIndexCorruption(err) || errors.Cause(err) != table[i].err {
+					t.Fatalf("expected %v, got %v", table[i].err, err)
+				}
+			})
+		}
+	})
+
+}
+
+func getSampleTree() (art.Tree, int) {
+	at := art.New()
+	keys := [][]byte{[]byte("abcd"), []byte("abce"), []byte("abcf"), []byte("abgd")}
+	expectedSerializedSize := 0
+	for i := range keys {
+		at.Insert(keys[i], internal.Item{FileID: i, Offset: int64(i), Size: int64(i)})
+		expectedSerializedSize += int32Size + len(keys[i]) + fileIDSize + offsetSize + sizeSize
+	}
+
+	return at, expectedSerializedSize
+}
--- a/internal/index/index.go
+++ b/internal/index/index.go
@@ -0,0 +1,59 @@
+package index
+
+import (
+	"os"
+
+	art "github.com/plar/go-adaptive-radix-tree"
+	"github.com/prologic/bitcask/internal"
+)
+
+// Indexer is an interface for loading and saving the index (an Adaptive Radix Tree)
+type Indexer interface {
+	Load(path string, maxkeySize uint32) (art.Tree, bool, error)
+	Save(t art.Tree, path string) error
+}
+
+// NewIndexer returns an instance of the default `Indexer` implemtnation
+// which perists the index (an Adaptive Radix Tree) as a binary blob on file
+func NewIndexer() Indexer {
+	return &indexer{}
+}
+
+type indexer struct{}
+
+func (i *indexer) Load(path string, maxKeySize uint32) (art.Tree, bool, error) {
+	t := art.New()
+
+	if !internal.Exists(path) {
+		return t, false, nil
+	}
+
+	f, err := os.Open(path)
+	if err != nil {
+		return t, true, err
+	}
+	defer f.Close()
+
+	if err := readIndex(f, t, maxKeySize); err != nil {
+		return t, true, err
+	}
+	return t, true, nil
+}
+
+func (i *indexer) Save(t art.Tree, path string) error {
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	if err := writeIndex(t, f); err != nil {
+		return err
+	}
+
+	if err := f.Sync(); err != nil {
+		return err
+	}
+
+	return f.Close()
+}
--- a/internal/item.go
+++ b/internal/item.go
@@ -1,5 +1,8 @@
 package internal

+// Item represents the location of the value on disk. This is used by the
+// internal Adaptive Radix Tree to hold an in-memory structure mapping keys to
+// locations on disk of where the value(s) can be read from.
 type Item struct {
 	FileID int   `json:"fileid"`
 	Offset int64 `json:"offset"`
--- a/internal/metadata/metadata.go
+++ b/internal/metadata/metadata.go
@@ -0,0 +1,22 @@
+package metadata
+
+import (
+	"os"
+
+	"github.com/prologic/bitcask/internal"
+)
+
+type MetaData struct {
+	IndexUpToDate    bool  `json:"index_up_to_date"`
+	ReclaimableSpace int64 `json:"reclaimable_space"`
+}
+
+func (m *MetaData) Save(path string, mode os.FileMode) error {
+	return internal.SaveJsonToFile(m, path, mode)
+}
+
+func Load(path string) (*MetaData, error) {
+	var m MetaData
+	err := internal.LoadFromJsonFile(path, &m)
+	return &m, err
+}
--- a/internal/mocks/datafile.go
+++ b/internal/mocks/datafile.go
@@ -0,0 +1,158 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+package mocks
+
+import internal "github.com/prologic/bitcask/internal"
+import mock "github.com/stretchr/testify/mock"
+
+// Datafile is an autogenerated mock type for the Datafile type
+type Datafile struct {
+	mock.Mock
+}
+
+// Close provides a mock function with given fields:
+func (_m *Datafile) Close() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// FileID provides a mock function with given fields:
+func (_m *Datafile) FileID() int {
+	ret := _m.Called()
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func() int); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	return r0
+}
+
+// Name provides a mock function with given fields:
+func (_m *Datafile) Name() string {
+	ret := _m.Called()
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func() string); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	return r0
+}
+
+// Read provides a mock function with given fields:
+func (_m *Datafile) Read() (internal.Entry, int64, error) {
+	ret := _m.Called()
+
+	var r0 internal.Entry
+	if rf, ok := ret.Get(0).(func() internal.Entry); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(internal.Entry)
+	}
+
+	var r1 int64
+	if rf, ok := ret.Get(1).(func() int64); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Get(1).(int64)
+	}
+
+	var r2 error
+	if rf, ok := ret.Get(2).(func() error); ok {
+		r2 = rf()
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
+}
+
+// ReadAt provides a mock function with given fields: index, size
+func (_m *Datafile) ReadAt(index int64, size int64) (internal.Entry, error) {
+	ret := _m.Called(index, size)
+
+	var r0 internal.Entry
+	if rf, ok := ret.Get(0).(func(int64, int64) internal.Entry); ok {
+		r0 = rf(index, size)
+	} else {
+		r0 = ret.Get(0).(internal.Entry)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(int64, int64) error); ok {
+		r1 = rf(index, size)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Size provides a mock function with given fields:
+func (_m *Datafile) Size() int64 {
+	ret := _m.Called()
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func() int64); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	return r0
+}
+
+// Sync provides a mock function with given fields:
+func (_m *Datafile) Sync() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Write provides a mock function with given fields: _a0
+func (_m *Datafile) Write(_a0 internal.Entry) (int64, int64, error) {
+	ret := _m.Called(_a0)
+
+	var r0 int64
+	if rf, ok := ret.Get(0).(func(internal.Entry) int64); ok {
+		r0 = rf(_a0)
+	} else {
+		r0 = ret.Get(0).(int64)
+	}
+
+	var r1 int64
+	if rf, ok := ret.Get(1).(func(internal.Entry) int64); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Get(1).(int64)
+	}
+
+	var r2 error
+	if rf, ok := ret.Get(2).(func(internal.Entry) error); ok {
+		r2 = rf(_a0)
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
+}
--- a/internal/mocks/indexer.go
+++ b/internal/mocks/indexer.go
@@ -0,0 +1,56 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+package mocks
+
+import art "github.com/plar/go-adaptive-radix-tree"
+
+import mock "github.com/stretchr/testify/mock"
+
+// Indexer is an autogenerated mock type for the Indexer type
+type Indexer struct {
+	mock.Mock
+}
+
+// Load provides a mock function with given fields: path, maxkeySize
+func (_m *Indexer) Load(path string, maxkeySize uint32) (art.Tree, bool, error) {
+	ret := _m.Called(path, maxkeySize)
+
+	var r0 art.Tree
+	if rf, ok := ret.Get(0).(func(string, uint32) art.Tree); ok {
+		r0 = rf(path, maxkeySize)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(art.Tree)
+		}
+	}
+
+	var r1 bool
+	if rf, ok := ret.Get(1).(func(string, uint32) bool); ok {
+		r1 = rf(path, maxkeySize)
+	} else {
+		r1 = ret.Get(1).(bool)
+	}
+
+	var r2 error
+	if rf, ok := ret.Get(2).(func(string, uint32) error); ok {
+		r2 = rf(path, maxkeySize)
+	} else {
+		r2 = ret.Error(2)
+	}
+
+	return r0, r1, r2
+}
+
+// Save provides a mock function with given fields: t, path
+func (_m *Indexer) Save(t art.Tree, path string) error {
+	ret := _m.Called(t, path)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(art.Tree, string) error); ok {
+		r0 = rf(t, path)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
--- a/internal/utils.go
+++ b/internal/utils.go
@@ -1,7 +1,9 @@
 package internal

 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
@@ -9,11 +11,14 @@ import (
 	"strings"
 )

+// Exists returns `true` if the given `path` on the current file system exists
 func Exists(path string) bool {
 	_, err := os.Stat(path)
 	return err == nil
 }

+// DirSize returns the space occupied by the given `path` on disk on the current
+// file system.
 func DirSize(path string) (int64, error) {
 	var size int64
 	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
@@ -28,6 +33,10 @@ func DirSize(path string) (int64, error) {
 	return size, err
 }

+// GetDatafiles returns a list of all data files stored in the database path
+// given by `path`. All datafiles are identified by the the glob `*.data` and
+// the basename is represented by a monotonic increasing integer.
+// The returned files are *sorted* in increasing order.
 func GetDatafiles(path string) ([]string, error) {
 	fns, err := filepath.Glob(fmt.Sprintf("%s/*.data", path))
 	if err != nil {
@@ -37,6 +46,8 @@ func GetDatafiles(path string) ([]string, error) {
 	return fns, nil
 }

+// ParseIds will parse a list of datafiles as returned by `GetDatafiles` and
+// extract the id part and return a slice of ints.
 func ParseIds(fns []string) ([]int, error) {
 	var ids []int
 	for _, fn := range fns {
@@ -54,3 +65,48 @@ func ParseIds(fns []string) ([]int, error) {
 	sort.Ints(ids)
 	return ids, nil
 }
+
+// Copy copies source contents to destination
+func Copy(src, dst string, exclude []string) error {
+	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+		relPath := strings.Replace(path, src, "", 1)
+		if relPath == "" {
+			return nil
+		}
+		for _, e := range exclude {
+			matched, err := filepath.Match(e, info.Name())
+			if err != nil {
+				return err
+			}
+			if matched {
+				return nil
+			}
+		}
+		if info.IsDir() {
+			return os.Mkdir(filepath.Join(dst, relPath), info.Mode())
+		}
+		var data, err1 = ioutil.ReadFile(filepath.Join(src, relPath))
+		if err1 != nil {
+			return err1
+		}
+		return ioutil.WriteFile(filepath.Join(dst, relPath), data, info.Mode())
+	})
+}
+
+// SaveJsonToFile converts v into json and store in file identified by path
+func SaveJsonToFile(v interface{}, path string, mode os.FileMode) error {
+	b, err := json.Marshal(v)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(path, b, mode)
+}
+
+// LoadFromJsonFile reads file located at `path` and put its content in json format in v
+func LoadFromJsonFile(path string, v interface{}) error {
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(b, v)
+}
--- a/internal/utils_test.go
+++ b/internal/utils_test.go
@@ -0,0 +1,108 @@
+package internal
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_Copy(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("CopyDir", func(t *testing.T) {
+		tempsrc, err := ioutil.TempDir("", "test")
+		assert.NoError(err)
+		defer os.RemoveAll(tempsrc)
+		var f *os.File
+
+		tempdir, err := ioutil.TempDir(tempsrc, "")
+		assert.NoError(err)
+
+		f, err = os.OpenFile(filepath.Join(tempsrc, "file1"), os.O_WRONLY|os.O_CREATE, 0755)
+		assert.NoError(err)
+		n, err := f.WriteString("test123")
+		assert.Equal(7, n)
+		assert.NoError(err)
+		f.Close()
+
+		f, err = os.OpenFile(filepath.Join(tempsrc, "file2"), os.O_WRONLY|os.O_CREATE, 0755)
+		assert.NoError(err)
+		n, err = f.WriteString("test1234")
+		assert.Equal(8, n)
+		assert.NoError(err)
+		f.Close()
+
+		f, err = os.OpenFile(filepath.Join(tempsrc, "file3"), os.O_WRONLY|os.O_CREATE, 0755)
+		assert.NoError(err)
+		f.Close()
+
+		tempdst, err := ioutil.TempDir("", "backup")
+		assert.NoError(err)
+		defer os.RemoveAll(tempdst)
+		err = Copy(tempsrc, tempdst, []string{"file3"})
+		assert.NoError(err)
+		buf := make([]byte, 10)
+
+		exists := Exists(filepath.Join(tempdst, filepath.Base(tempdir)))
+		assert.Equal(true, exists)
+
+		f, err = os.Open(filepath.Join(tempdst, "file1"))
+		assert.NoError(err)
+		n, err = f.Read(buf[:7])
+		assert.NoError(err)
+		assert.Equal(7, n)
+		assert.Equal([]byte("test123"), buf[:7])
+		_, err = f.Read(buf)
+		assert.Equal(io.EOF, err)
+		f.Close()
+
+		f, err = os.Open(filepath.Join(tempdst, "file2"))
+		assert.NoError(err)
+		n, err = f.Read(buf[:8])
+		assert.NoError(err)
+		assert.Equal(8, n)
+		assert.Equal([]byte("test1234"), buf[:8])
+		_, err = f.Read(buf)
+		assert.Equal(io.EOF, err)
+		f.Close()
+
+		exists = Exists(filepath.Join(tempdst, "file3"))
+		assert.Equal(false, exists)
+	})
+}
+
+func Test_SaveAndLoad(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("save and load", func(t *testing.T) {
+		tempdir, err := ioutil.TempDir("", "bitcask")
+		assert.NoError(err)
+		defer os.RemoveAll(tempdir)
+		type test struct {
+			Value bool `json:"value"`
+		}
+		m := test{Value: true}
+		err = SaveJsonToFile(&m, filepath.Join(tempdir, "meta.json"), 0755)
+		assert.NoError(err)
+		m1 := test{}
+		err = LoadFromJsonFile(filepath.Join(tempdir, "meta.json"), &m1)
+		assert.NoError(err)
+		assert.Equal(m, m1)
+	})
+
+	t.Run("save and load error", func(t *testing.T) {
+		tempdir, err := ioutil.TempDir("", "bitcask")
+		assert.NoError(err)
+		defer os.RemoveAll(tempdir)
+		type test struct {
+			Value bool `json:"value"`
+		}
+		err = SaveJsonToFile(make(chan int), filepath.Join(tempdir, "meta.json"), 0755)
+		assert.Error(err)
+		m1 := test{}
+		err = LoadFromJsonFile(filepath.Join(tempdir, "meta.json"), &m1)
+		assert.Error(err)
+	})
+}
--- a/options.go
+++ b/options.go
@@ -1,101 +1,95 @@
 package bitcask

 import (
-	"encoding/json"
-	"io/ioutil"
-	"path/filepath"
+	"os"
+
+	"github.com/prologic/bitcask/internal/config"
 )

 const (
+	// DefaultDirFileModeBeforeUmask is the default os.FileMode used when creating directories
+	DefaultDirFileModeBeforeUmask = os.FileMode(0700)
+
+	// DefaultFileFileModeBeforeUmask is the default os.FileMode used when creating files
+	DefaultFileFileModeBeforeUmask = os.FileMode(0600)
+
 	// DefaultMaxDatafileSize is the default maximum datafile size in bytes
 	DefaultMaxDatafileSize = 1 << 20 // 1MB

 	// DefaultMaxKeySize is the default maximum key size in bytes
-	DefaultMaxKeySize = 64 // 64 bytes
+	DefaultMaxKeySize = uint32(64) // 64 bytes

 	// DefaultMaxValueSize is the default value size in bytes
-	DefaultMaxValueSize = 1 << 16 // 65KB
+	DefaultMaxValueSize = uint64(1 << 16) // 65KB
+
+	// DefaultSync is the default file synchronization action
+	DefaultSync = false
+
+	// DefaultAutoRecovery is the default auto-recovery action.
 )

 // Option is a function that takes a config struct and modifies it
-type Option func(*config) error
+type Option func(*config.Config) error

-type config struct {
-	maxDatafileSize int
-	maxKeySize      int
-	maxValueSize    int
-	sync            bool
+func withConfig(src *config.Config) Option {
+	return func(cfg *config.Config) error {
+		cfg.MaxDatafileSize = src.MaxDatafileSize
+		cfg.MaxKeySize = src.MaxKeySize
+		cfg.MaxValueSize = src.MaxValueSize
+		cfg.Sync = src.Sync
+		cfg.AutoRecovery = src.AutoRecovery
+		cfg.DirFileModeBeforeUmask = src.DirFileModeBeforeUmask
+		cfg.FileFileModeBeforeUmask = src.FileFileModeBeforeUmask
+		return nil
+	}
 }

-func (c *config) MarshalJSON() ([]byte, error) {
-	return json.Marshal(struct {
-		MaxDatafileSize int  `json:"max_datafile_size"`
-		MaxKeySize      int  `json:"max_key_size"`
-		MaxValueSize    int  `json:"max_value_size"`
-		Sync            bool `json:"sync"`
-	}{
-		MaxDatafileSize: c.maxDatafileSize,
-		MaxKeySize:      c.maxKeySize,
-		MaxValueSize:    c.maxValueSize,
-		Sync:            c.sync,
-	})
+// WithAutoRecovery sets auto recovery of data and index file recreation.
+// IMPORTANT: This flag MUST BE used only if a proper backup was made of all
+// the existing datafiles.
+func WithAutoRecovery(enabled bool) Option {
+	return func(cfg *config.Config) error {
+		cfg.AutoRecovery = enabled
+		return nil
+	}
 }

-func getConfig(path string) (*config, error) {
-	type Config struct {
-		MaxDatafileSize int  `json:"max_datafile_size"`
-		MaxKeySize      int  `json:"max_key_size"`
-		MaxValueSize    int  `json:"max_value_size"`
-		Sync            bool `json:"sync"`
+// WithDirFileModeBeforeUmask sets the FileMode used for each new file created.
+func WithDirFileModeBeforeUmask(mode os.FileMode) Option {
+	return func(cfg *config.Config) error {
+		cfg.DirFileModeBeforeUmask = mode
+		return nil
 	}
-
-	var cfg Config
-
-	data, err := ioutil.ReadFile(filepath.Join(path, "config.json"))
-	if err != nil {
-		return nil, err
-	}
-
-	if err := json.Unmarshal(data, &cfg); err != nil {
-		return nil, err
-	}
-
-	return &config{
-		maxDatafileSize: cfg.MaxDatafileSize,
-		maxKeySize:      cfg.MaxKeySize,
-		maxValueSize:    cfg.MaxValueSize,
-		sync:            cfg.Sync,
-	}, nil
 }

-func newDefaultConfig() *config {
-	return &config{
-		maxDatafileSize: DefaultMaxDatafileSize,
-		maxKeySize:      DefaultMaxKeySize,
-		maxValueSize:    DefaultMaxValueSize,
+// WithFileFileModeBeforeUmask sets the FileMode used for each new file created.
+func WithFileFileModeBeforeUmask(mode os.FileMode) Option {
+	return func(cfg *config.Config) error {
+		cfg.FileFileModeBeforeUmask = mode
+		return nil
 	}
 }

 // WithMaxDatafileSize sets the maximum datafile size option
 func WithMaxDatafileSize(size int) Option {
-	return func(cfg *config) error {
-		cfg.maxDatafileSize = size
+	return func(cfg *config.Config) error {
+		cfg.MaxDatafileSize = size
 		return nil
 	}
 }

 // WithMaxKeySize sets the maximum key size option
-func WithMaxKeySize(size int) Option {
-	return func(cfg *config) error {
-		cfg.maxKeySize = size
+func WithMaxKeySize(size uint32) Option {
+	return func(cfg *config.Config) error {
+		cfg.MaxKeySize = size
 		return nil
 	}
 }

 // WithMaxValueSize sets the maximum value size option
-func WithMaxValueSize(size int) Option {
-	return func(cfg *config) error {
-		cfg.maxValueSize = size
+func WithMaxValueSize(size uint64) Option {
+	return func(cfg *config.Config) error {
+		cfg.MaxValueSize = size
 		return nil
 	}
 }
@@ -103,8 +97,19 @@ func WithMaxValueSize(size int) Option {
 // WithSync causes Sync() to be called on every key/value written increasing
 // durability and safety at the expense of performance
 func WithSync(sync bool) Option {
-	return func(cfg *config) error {
-		cfg.sync = sync
+	return func(cfg *config.Config) error {
+		cfg.Sync = sync
 		return nil
 	}
 }
+
+func newDefaultConfig() *config.Config {
+	return &config.Config{
+		MaxDatafileSize:         DefaultMaxDatafileSize,
+		MaxKeySize:              DefaultMaxKeySize,
+		MaxValueSize:            DefaultMaxValueSize,
+		Sync:                    DefaultSync,
+		DirFileModeBeforeUmask:  DefaultDirFileModeBeforeUmask,
+		FileFileModeBeforeUmask: DefaultFileFileModeBeforeUmask,
+	}
+}
--- a/tools/release.sh
+++ b/tools/release.sh
@@ -1,16 +1,19 @@
-#!/bin/sh
+#!/bin/bash

 # Get the highest tag number
 VERSION="$(git describe --abbrev=0 --tags)"
 VERSION=${VERSION:-'0.0.0'}

 # Get number parts
-MAJOR="${VERSION%%.*}"; VERSION="${VERSION#*.}"
-MINOR="${VERSION%%.*}"; VERSION="${VERSION#*.}"
-PATCH="${VERSION%%.*}"; VERSION="${VERSION#*.}"
+MAJOR="${VERSION%%.*}"
+VERSION="${VERSION#*.}"
+MINOR="${VERSION%%.*}"
+VERSION="${VERSION#*.}"
+PATCH="${VERSION%%.*}"
+VERSION="${VERSION#*.}"

 # Increase version
-PATCH=$((PATCH+1))
+PATCH=$((PATCH + 1))

 TAG="${1}"

@@ -20,6 +23,10 @@ fi

 echo "Releasing ${TAG} ..."

+git-chglog --next-tag="${TAG}" --output CHANGELOG.md
+git commit -a -m "Update CHANGELOG for ${TAG}"
 git tag -a -s -m "Release ${TAG}" "${TAG}"
 git push --tags
-goreleaser release --rm-dist
+goreleaser release \
+  --rm-dist \
+  --release-notes <(git-chglog "${TAG}" | tail -n+5)