Update CHANGELOG for v0.3.13

Supercesd #219 after rebasing on master after migrating off Github.

Co-authored-by: Nicolò Santamaria <nicolo.santamaria@protonmail.com>
Co-authored-by: James Mills <prologic@shortcircuit.net.au>
Co-authored-by: Tai Groot <taigrr@noreply@mills.io>
Reviewed-on: https://git.mills.io/prologic/bitcask/pulls/224
Co-authored-by: James Mills <prologic@noreply@mills.io>
Co-committed-by: James Mills <prologic@noreply@mills.io>

.chglog/CHANGELOG.tpl.md

@@ -0,0 +1,22 @@
+{{ range .Versions }}
+<a name="{{ .Tag.Name }}"></a>
+## {{ if .Tag.Previous }}[{{ .Tag.Name }}]({{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}){{ else }}{{ .Tag.Name }}{{ end }} ({{ datetime "2006-01-02" .Tag.Date }})
+
+{{ range .CommitGroups -}}
+### {{ .Title }}
+
+{{ range .Commits -}}
+* {{ .Subject }}
+{{ end }}
+{{ end -}}
+
+{{- if .NoteGroups -}}
+{{ range .NoteGroups -}}
+### {{ .Title }}
+
+{{ range .Notes }}
+{{ .Body }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+{{ end -}}

.chglog/config.yml

@@ -0,0 +1,28 @@
+---
+style: Github
+template: CHANGELOG.tpl.md
+info:
+  title: CHANGELOG
+  repository_url: https://git.mills.io/prologic/bitcask
+options:
+  commits:
+    filters:
+      Type:
+        - Add
+        - Fix
+        - Update
+        - Document
+  commit_groups:
+    title_maps:
+      Add: Features
+      Update: Updates
+      Fix: Bug Fixes
+      Document: Documentation
+  header:
+    pattern: "^((\\w+)\\s.*)$"
+    pattern_maps:
+      - Subject
+      - Type
+  notes:
+    keywords:
+      - BREAKING CHANGE

.drone.yml

@@ -1,14 +1,31 @@
+---
 kind: pipeline
 name: default
 
 steps:
-  - name: build
-    image: golang:latest
+  - name: build & run tests
+    image: r.mills.io/prologic/golang-alpine
+    volumes:
+      - name: gomodcache
+        path: /go/pkg/mod/cache
+    privileged: true
+    environment:
+      GOPROXY: https://goproxy.mills.io
     commands:
-      - go test -v -cover -coverprofile=coverage.txt -covermode=atomic -coverpkg=$(shell go list) -race .
+      - make build
+      - make test
 
-  - name: coverage
-    image: plugins/codecov
+  - name: notify
+    image: plugins/webhook
     settings:
-      token:
-        from_secret: codecov-token
+      urls:
+        - https://msgbus.mills.io/ci.mills.io
+    when:
+      status:
+        - success
+        - failure
+
+volumes:
+  - name: gomodcache
+    host:
+      path: /var/lib/cache/go

.github/FUNDING.yml

@@ -1,2 +0,0 @@
-github: prologic
-patreon: prologic

.gitignore

@@ -1,9 +1,14 @@
 *~*
-*.bak
 *.db
+*.bak
+**.envrc
 
-/coverage.txt
-/bitcask
-/bitcaskd
 /tmp
 /dist
+/coverage.txt
+
+/bitcask
+/bitcaskd
+/bitcask_bench*
+/cmd/bitcask/bitcask
+/cmd/bitcaskd/bitcaskd

.goreleaser.yml

@@ -1,34 +1,39 @@
+---
 builds:
-  - 
+  -
+    id: bitcask
     binary: bitcask
     main: ./cmd/bitcask
     flags: -tags "static_build"
-    ldflags: -w -X github.com/prologic/bitcask/internal.Version={{.Version}} -X github.com/prologic/bitcask/internal.Commit={{.Commit}}
+    ldflags: -w -X git.mills.io/prologic/bitcask/internal.Version={{.Version}} -X git.mills.io/prologic/bitcask/internal.Commit={{.Commit}}
     env:
       - CGO_ENABLED=0
-  - 
+    goos:
+      - darwin
+      - linux
+    goarch:
+      - amd64
+      - arm64
+  -
+    id: bitcaskd
     binary: bitcaskd
     main: ./cmd/bitcaskd
     flags: -tags "static_build"
-    ldflags: -w -X github.com/prologic/bitcask/internal.Version={{.Version}} -X github.com/prologic/bitcask/internal.Commit={{.Commit}}
+    ldflags: -w -X git.mills.io/prologic/bitcask/internal.Version={{.Version}} -X git.mills.io/prologic/bitcask/internal.Commit={{.Commit}}
     env:
       - CGO_ENABLED=0
-sign:
-  artifacts: checksum
-archive:
-  replacements:
-    darwin: Darwin
-    linux: Linux
-    windows: Windows
-    386: i386
-    amd64: x86_64
-checksum:
-  name_template: 'checksums.txt'
-snapshot:
-  name_template: "{{ .Tag }}-next"
-changelog:
-  sort: asc
-  filters:
-    exclude:
-    - '^docs:'
-    - '^test:'
+    goos:
+      - darwin
+      - linux
+    goarch:
+      - amd64
+      - arm64
+signs:
+  - artifacts: checksum
+release:
+  gitea:
+    owner: prologic
+    name: bitcask
+  draft: true
+gitea_urls:
+  api: https://git.mills.io/api/v1/

AUTHORS

@@ -2,12 +2,18 @@
 #   Name or Organization <email address>
 # The email address is not required for organizations.
 
+Alain Gilbert <alain.gilbert.15@gmail.com>
 Awn Umar <awn@spacetime.dev>
+Bryan Stenson <bryan@siliconvortex.com>
 Christian Muehlhaeuser <muesli@gmail.com>
 Ignacio Hagopian <jsign.uy@gmail.com>
 James Mills <prologic@shortcircuit.net.au>
 Jesse Donat <donatj@gmail.com>
 Kebert Xela kebertxela
 panyun panyun 
+shiniao <zhuzhezhe95@gmail.com>
 Whemoon Jang <palindrom615@gmail.com>
+Yash Chandra <yashschandra@gmail.com>
 Yury Fedorov orlangure
+o2gy84 <o2gy84@gmail.com>
+garsue <labs.garsue@gmail.com>

CHANGELOG.md

@@ -0,0 +1,437 @@
+
+<a name="v0.3.13"></a>
+## [v0.3.13](https://git.mills.io/prologic/bitcask/compare/v0.3.12...v0.3.13) (2021-07-16)
+
+### Bug Fixes
+
+* Fix paths used for temporary recovery iles to avoid crossing devices (#223)
+
+### Features
+
+* Add Drone CI config
+
+
+<a name="v0.3.12"></a>
+## [v0.3.12](https://git.mills.io/prologic/bitcask/compare/v0.3.11...v0.3.12) (2021-07-13)
+
+### Updates
+
+* Update CHANGELOG for v0.3.12
+
+
+<a name="v0.3.11"></a>
+## [v0.3.11](https://git.mills.io/prologic/bitcask/compare/v0.3.10...v0.3.11) (2021-07-10)
+
+### Bug Fixes
+
+* Fix missing go.sum entries
+* Fix GoReleaser config
+* Fix go.sum
+
+### Documentation
+
+* Document good and possibly poor use-cases of Bitcask (#199)
+
+### Features
+
+* Add support for keys with ttl (#177)
+
+### Updates
+
+* Update CHANGELOG for v0.3.11
+
+
+<a name="v0.3.10"></a>
+## [v0.3.10](https://git.mills.io/prologic/bitcask/compare/v0.3.9...v0.3.10) (2020-12-18)
+
+### Bug Fixes
+
+* Fix a bug when MaxValueSize == 0 on Merge operations
+* Fix link to bitcask-bench
+* Fix CI (again)
+* Fix CI
+
+### Features
+
+* Add support for unlimited key/value sizes
+* Add a few more test cases for concurrent operations
+
+### Updates
+
+* Update CHANGELOG for v0.3.10
+* Update README.md
+
+
+<a name="v0.3.9"></a>
+## [v0.3.9](https://git.mills.io/prologic/bitcask/compare/v0.3.8...v0.3.9) (2020-11-17)
+
+### Bug Fixes
+
+* Fix a race condition around .Close() and .Sync()
+
+### Updates
+
+* Update CHANGELOG for v0.3.9
+
+
+<a name="v0.3.8"></a>
+## [v0.3.8](https://git.mills.io/prologic/bitcask/compare/v0.3.7...v0.3.8) (2020-11-17)
+
+### Updates
+
+* Update CHANGELOG for v0.3.8
+
+
+<a name="v0.3.7"></a>
+## [v0.3.7](https://git.mills.io/prologic/bitcask/compare/v0.3.6...v0.3.7) (2020-11-17)
+
+### Updates
+
+* Update CHANGELOG for v0.3.7
+
+
+<a name="v0.3.6"></a>
+## [v0.3.6](https://git.mills.io/prologic/bitcask/compare/v0.3.5...v0.3.6) (2020-11-17)
+
+### Bug Fixes
+
+* Fix typo in labeler (#172)
+* Fix builds configuration for goreleaser
+* Fix (again) goreleaser config
+* Fix goreleaser config and improve release notes / changelog
+* Fix recoverDatafile error covering (#162)
+* Fix loadIndex to be deterministic (#115)
+
+### Features
+
+* Add configuration options for FileMode (#183)
+* Add imports and log in example code (#182)
+* Add empty changelog
+* Add DependaBot config
+* Add DeleteAll function (#116)
+
+### Updates
+
+* Update CHANGELOG for v0.3.6
+* Update README.md
+* Update CHANGELOG for v0.3.6
+* Update CHANGELOG for v0.3.6
+* Update deps (#140)
+* Update README.md
+
+
+<a name="v0.3.5"></a>
+## [v0.3.5](https://git.mills.io/prologic/bitcask/compare/v0.3.4...v0.3.5) (2019-10-20)
+
+### Bug Fixes
+
+* Fix setup target in Makefile to install mockery correctly
+* Fix glfmt/golint issues
+* Fix spelling mistake in README s/Sponser/Sponsor
+
+### Features
+
+* Add *.db to ignore future accidental commits of a bitcask db to the repo
+* Add unit test for opening bad database with corrupted/invalid datafiles (#105)
+
+### Updates
+
+* Update Drone CI test pipeline
+* Update README.md
+* Update to Go 1.13 and update README with new benchmarks (#89)
+* Update README.md
+
+
+<a name="v0.3.4"></a>
+## [v0.3.4](https://git.mills.io/prologic/bitcask/compare/v0.3.3...v0.3.4) (2019-09-02)
+
+
+<a name="v0.3.3"></a>
+## [v0.3.3](https://git.mills.io/prologic/bitcask/compare/v0.3.2...v0.3.3) (2019-09-02)
+
+### Bug Fixes
+
+* Fix a bug wit the decoder passing the wrong value for the value's offset into the buffer (#77)
+* Fix typo (#65)
+* Fix and cleanup some unnecessary internal sub-packages and duplication
+
+### Updates
+
+* Update README.md
+* Update README.md
+* Update README.md
+* Update README.md
+
+
+<a name="v0.3.2"></a>
+## [v0.3.2](https://git.mills.io/prologic/bitcask/compare/v0.3.1...v0.3.2) (2019-08-08)
+
+### Updates
+
+* Update README.md
+* Update README.md
+* Update CONTRIBUTING.md
+
+
+<a name="v0.3.1"></a>
+## [v0.3.1](https://git.mills.io/prologic/bitcask/compare/v0.3.0...v0.3.1) (2019-08-05)
+
+### Updates
+
+* Update README.md
+* Update README.md
+* Update README.md
+
+
+<a name="v0.3.0"></a>
+## [v0.3.0](https://git.mills.io/prologic/bitcask/compare/v0.2.2...v0.3.0) (2019-07-29)
+
+### Updates
+
+* Update README.md
+* Update README.md
+
+
+<a name="v0.2.2"></a>
+## [v0.2.2](https://git.mills.io/prologic/bitcask/compare/v0.2.1...v0.2.2) (2019-07-27)
+
+
+<a name="v0.2.1"></a>
+## [v0.2.1](https://git.mills.io/prologic/bitcask/compare/v0.2.0...v0.2.1) (2019-07-25)
+
+
+<a name="v0.2.0"></a>
+## [v0.2.0](https://git.mills.io/prologic/bitcask/compare/v0.1.7...v0.2.0) (2019-07-25)
+
+### Bug Fixes
+
+* Fix issue(db file Merge issue in windows env): (#15)
+
+
+<a name="v0.1.7"></a>
+## [v0.1.7](https://git.mills.io/prologic/bitcask/compare/v0.1.6...v0.1.7) (2019-07-19)
+
+### Bug Fixes
+
+* Fix mismatched key casing. (#12)
+* Fix outdated README (#11)
+* Fix typos in bitcask.go docs (#10)
+
+### Updates
+
+* Update generated protobuf code
+* Update README.md
+
+
+<a name="v0.1.6"></a>
+## [v0.1.6](https://git.mills.io/prologic/bitcask/compare/v0.1.5...v0.1.6) (2019-04-01)
+
+### Features
+
+* Add Development section to README documenting use of Protobuf and tooling required. #6
+* Add other badges from img.shields.io
+
+
+<a name="v0.1.5"></a>
+## [v0.1.5](https://git.mills.io/prologic/bitcask/compare/v0.1.4...v0.1.5) (2019-03-30)
+
+### Documentation
+
+* Document using the Docker Image
+
+### Features
+
+* Add Dockerfile to publish images to Docker Hub
+
+### Updates
+
+* Update README.md
+
+
+<a name="v0.1.4"></a>
+## [v0.1.4](https://git.mills.io/prologic/bitcask/compare/v0.1.3...v0.1.4) (2019-03-23)
+
+
+<a name="v0.1.3"></a>
+## [v0.1.3](https://git.mills.io/prologic/bitcask/compare/v0.1.2...v0.1.3) (2019-03-23)
+
+
+<a name="v0.1.2"></a>
+## [v0.1.2](https://git.mills.io/prologic/bitcask/compare/v0.1.1...v0.1.2) (2019-03-22)
+
+
+<a name="v0.1.1"></a>
+## [v0.1.1](https://git.mills.io/prologic/bitcask/compare/v0.1.0...v0.1.1) (2019-03-22)
+
+
+<a name="v0.1.0"></a>
+## [v0.1.0](https://git.mills.io/prologic/bitcask/compare/0.0.26...v0.1.0) (2019-03-22)
+
+
+<a name="0.0.26"></a>
+## [0.0.26](https://git.mills.io/prologic/bitcask/compare/0.0.25...0.0.26) (2019-03-21)
+
+### Features
+
+* Add docs for bitcask
+* Add docs for options
+* Add KeYS command to server (bitraftd)
+* Add Len() to exported API (extended API)
+* Add Keys() to exported API (extended API)
+* Add EXISTS command to server (bitraftd)
+
+
+<a name="0.0.25"></a>
+## [0.0.25](https://git.mills.io/prologic/bitcask/compare/0.0.24...0.0.25) (2019-03-21)
+
+### Features
+
+* Add Has() to exported API (extended API)
+* Add MergeOpen test case
+
+### Updates
+
+* Update README.md
+* Update README.md
+
+
+<a name="0.0.24"></a>
+## [0.0.24](https://git.mills.io/prologic/bitcask/compare/0.0.23...0.0.24) (2019-03-20)
+
+
+<a name="0.0.23"></a>
+## [0.0.23](https://git.mills.io/prologic/bitcask/compare/0.0.22...0.0.23) (2019-03-20)
+
+### Features
+
+* Add bitcaskd to install target
+
+
+<a name="0.0.22"></a>
+## [0.0.22](https://git.mills.io/prologic/bitcask/compare/0.0.21...0.0.22) (2019-03-18)
+
+
+<a name="0.0.21"></a>
+## [0.0.21](https://git.mills.io/prologic/bitcask/compare/0.0.20...0.0.21) (2019-03-18)
+
+
+<a name="0.0.20"></a>
+## [0.0.20](https://git.mills.io/prologic/bitcask/compare/0.0.19...0.0.20) (2019-03-17)
+
+
+<a name="0.0.19"></a>
+## [0.0.19](https://git.mills.io/prologic/bitcask/compare/0.0.18...0.0.19) (2019-03-17)
+
+
+<a name="0.0.18"></a>
+## [0.0.18](https://git.mills.io/prologic/bitcask/compare/0.0.17...0.0.18) (2019-03-16)
+
+
+<a name="0.0.17"></a>
+## [0.0.17](https://git.mills.io/prologic/bitcask/compare/0.0.16...0.0.17) (2019-03-16)
+
+### Features
+
+* Add CRC Checksum checks on reading values back
+
+
+<a name="0.0.16"></a>
+## [0.0.16](https://git.mills.io/prologic/bitcask/compare/0.0.15...0.0.16) (2019-03-16)
+
+
+<a name="0.0.15"></a>
+## [0.0.15](https://git.mills.io/prologic/bitcask/compare/0.0.14...0.0.15) (2019-03-16)
+
+### Bug Fixes
+
+* Fix a race condition + Use my fork of trie
+
+
+<a name="0.0.14"></a>
+## [0.0.14](https://git.mills.io/prologic/bitcask/compare/0.0.13...0.0.14) (2019-03-16)
+
+
+<a name="0.0.13"></a>
+## [0.0.13](https://git.mills.io/prologic/bitcask/compare/0.0.12...0.0.13) (2019-03-16)
+
+### Features
+
+* Add prefix scan for keys using a Trie
+
+
+<a name="0.0.12"></a>
+## [0.0.12](https://git.mills.io/prologic/bitcask/compare/0.0.11...0.0.12) (2019-03-14)
+
+
+<a name="0.0.11"></a>
+## [0.0.11](https://git.mills.io/prologic/bitcask/compare/0.0.10...0.0.11) (2019-03-14)
+
+### Updates
+
+* Update README.md
+
+
+<a name="0.0.10"></a>
+## [0.0.10](https://git.mills.io/prologic/bitcask/compare/0.0.9...0.0.10) (2019-03-14)
+
+### Bug Fixes
+
+* Fix concurrent read bug
+* Fix concurrent write bug with multiple goroutines writing to the to the active datafile
+
+### Updates
+
+* Update README.md
+
+
+<a name="0.0.9"></a>
+## [0.0.9](https://git.mills.io/prologic/bitcask/compare/0.0.8...0.0.9) (2019-03-14)
+
+
+<a name="0.0.8"></a>
+## [0.0.8](https://git.mills.io/prologic/bitcask/compare/0.0.7...0.0.8) (2019-03-13)
+
+
+<a name="0.0.7"></a>
+## [0.0.7](https://git.mills.io/prologic/bitcask/compare/0.0.6...0.0.7) (2019-03-13)
+
+
+<a name="0.0.6"></a>
+## [0.0.6](https://git.mills.io/prologic/bitcask/compare/0.0.5...0.0.6) (2019-03-13)
+
+### Bug Fixes
+
+* Fix usage output of bitcaskd
+
+
+<a name="0.0.5"></a>
+## [0.0.5](https://git.mills.io/prologic/bitcask/compare/0.0.4...0.0.5) (2019-03-13)
+
+### Features
+
+* Add a simple Redis compatible server daemon (bitcaskd)
+
+### Updates
+
+* Update README.md
+
+
+<a name="0.0.4"></a>
+## [0.0.4](https://git.mills.io/prologic/bitcask/compare/0.0.3...0.0.4) (2019-03-13)
+
+### Features
+
+* Add flock on database Open()/Close() to prevent multiple concurrent processes write access. Fixes #2
+
+
+<a name="0.0.3"></a>
+## [0.0.3](https://git.mills.io/prologic/bitcask/compare/0.0.2...0.0.3) (2019-03-13)
+
+
+<a name="0.0.2"></a>
+## [0.0.2](https://git.mills.io/prologic/bitcask/compare/0.0.1...0.0.2) (2019-03-13)
+
+
+<a name="0.0.1"></a>
+## 0.0.1 (2019-03-13)
+

Makefile

@@ -37,7 +37,7 @@ profile: build
 	@go test -cpuprofile cpu.prof -memprofile mem.prof -v -bench .
 
 bench: build
-	@go test -v -benchmem -bench=. .
+	@go test -v -run=XXX -benchmem -bench=. .
 
 mocks:
 	@mockery -all -case underscore -output ./internal/mocks -recursive

README.md

@@ -1,66 +1,129 @@
 # bitcask
 
-[![Build Status](https://cloud.drone.io/api/badges/prologic/bitcask/status.svg)](https://cloud.drone.io/prologic/bitcask)
+![](https://git.mills.io/prologic/bitcask/workflows/Coverage/badge.svg)
+![](https://git.mills.io/prologic/bitcask/workflows/Docker/badge.svg)
+![](https://git.mills.io/prologic/bitcask/workflows/Go/badge.svg)
+![](https://git.mills.io/prologic/bitcask/workflows/ReviewDog/badge.svg)
+
 [![CodeCov](https://codecov.io/gh/prologic/bitcask/branch/master/graph/badge.svg)](https://codecov.io/gh/prologic/bitcask)
 [![Go Report Card](https://goreportcard.com/badge/prologic/bitcask)](https://goreportcard.com/report/prologic/bitcask)
 [![codebeat badge](https://codebeat.co/badges/15fba8a5-3044-4f40-936f-9e0f5d5d1fd9)](https://codebeat.co/projects/github-com-prologic-bitcask-master)
-[![GoDoc](https://godoc.org/github.com/prologic/bitcask?status.svg)](https://godoc.org/github.com/prologic/bitcask) 
-[![GitHub license](https://img.shields.io/github/license/prologic/bitcask.svg)](https://github.com/prologic/bitcask)
-[![Sourcegraph](https://sourcegraph.com/github.com/prologic/bitcask/-/badge.svg)](https://sourcegraph.com/github.com/prologic/bitcask?badge)
+[![GoDoc](https://godoc.org/git.mills.io/prologic/bitcask?status.svg)](https://godoc.org/git.mills.io/prologic/bitcask) 
+[![GitHub license](https://img.shields.io/github/license/prologic/bitcask.svg)](https://git.mills.io/prologic/bitcask)
+[![Sourcegraph](https://sourcegraph.com/git.mills.io/prologic/bitcask/-/badge.svg)](https://sourcegraph.com/git.mills.io/prologic/bitcask?badge)
+[![TODOs](https://img.shields.io/endpoint?url=https://api.tickgit.com/badge?repo=git.mills.io/prologic/bitcask)](https://www.tickgit.com/browse?repo=git.mills.io/prologic/bitcask)
 
 A high performance Key/Value store written in [Go](https://golang.org) with a predictable read/write performance and high throughput. Uses a [Bitcask](https://en.wikipedia.org/wiki/Bitcask) on-disk layout (LSM+WAL) similar to [Riak](https://riak.com/)
 
-For a more feature-complete Redis-compatible server, distributed key/value store have a look at [Bitraft](https://github.com/prologic/bitraft) which uses this library as its backend. Use [Bitcask](https://github.com/prologic/bitcask) as a starting point or if you want to embed in your application, use [Bitraft](https://github.com/prologic/bitraft) if you need a complete server/client solution with high availability with a Redis-compatible API.
+For a more feature-complete Redis-compatible server, distributed key/value store have a look at [Bitraft](https://git.mills.io/prologic/bitraft) which uses this library as its backend. Use [Bitcask](https://git.mills.io/prologic/bitcask) as a starting point or if you want to embed in your application, use [Bitraft](https://git.mills.io/prologic/bitraft) if you need a complete server/client solution with high availability with a Redis-compatible API.
 
 ## Features
 
-* Embeddable (`import "github.com/prologic/bitcask"`)
+* Embedded (`import "git.mills.io/prologic/bitcask"`)
 * Builtin CLI (`bitcask`)
 * Builtin Redis-compatible server (`bitcaskd`)
 * Predictable read/write performance
 * Low latency
 * High throughput (See: [Performance](README.md#Performance) )
 
+## Is Bitcask right for my project?
+
+__NOTE__: Please read this carefully to identify whether using Bitcask is
+          suitable for your needs.
+
+`bitcask` is a **great fit** for:
+
+- Storing hundreds of thousands to millions of key/value pairs based on
+  default configuration. With the default configuration (_configurable_)
+  of 64 bytes per key and 64kB values, 1M keys would consume roughly ~600-700MB
+  of memory ~65-70GB of disk storage. These are all configurable when you
+  create a new database with `bitcask.Open(...)` with functional-style options
+  you can pass with `WithXXX()`.
+
+- As the backing store to a distributed key/value store. See for example the
+  [bitraft](https://git.mills.io/prologic/bitraft) as an example of this.
+
+- For high performance, low latency read/write workloads where you cannot fit
+  a typical hash-map into memory, but require the highest level of performance
+  and predicate read latency. Bitcask ensures only 1 read/write IOPS are ever
+  required for reading and writing key/value pairs.
+
+- As a general purpose embedded key/value store where you would have used
+  [BoltDB](https://github.com/boltdb/bolt),
+  [LevelDB](https://github.com/syndtr/goleveldb),
+  [BuntDB](https://github.com/tidwall/buntdb)
+  or similar...
+
+`bitcask` is not suited for:
+
+- Storing billions of records
+  The reason for this is the key-space is held in memory using a highly
+  performant and memory optimized adaptive radix tree thanks to
+  [go-adaptive-radix-tree](github.com/plar/go-adaptive-radix-tree) _however_
+  this means the more keys you have in your key space, the more memory is
+  consumed. Consider using a disk-backed B-Tree like [BoltDB](https://github.com/boltdb/bolt)
+  or [LevelDB](https://github.com/syndtr/goleveldb) if you intend to store a
+  large quantity of key/value pairs.
+
+> Note however that storing large amounts of data in terms of value(s) is
+> totally fine. In other wise thousands to millions of keys with large values
+> will work just fine.
+
+- Write intensive workloads. Due to the [Bitcask design](https://riak.com/assets/bitcask-intro.pdf?source=post_page---------------------------)
+  heavy write workloads that lots of key/value pairs will over time cause
+  problems like "Too many open files" (#193) errors to occur. This can be mitigated by
+  periodically compacting the data files by issuing a `.Merge()` operation however
+  if key/value pairs do not change or are never deleted, as-in only new key/value
+  pairs are ever written this will have no effect. Eventually you will run out
+  of file descriptors!
+
+> You should consider your read/write workloads carefully and ensure you set
+> appropriate file descriptor limits with `ulimit -n` that suit your needs.
+
 ## Development
 
-```#!sh
-$ git clone https://github.com/prologic/bitcask.git
+```sh
+$ git clone https://git.mills.io/prologic/bitcask.git
 $ make
 ```
 
 ## Install
 
-```#!sh
-$ go get github.com/prologic/bitcask
+```sh
+$ go get git.mills.io/prologic/bitcask
 ```
 
 ## Usage (library)
 
 Install the package into your project:
 
-```#!sh
-$ go get github.com/prologic/bitcask
+```sh
+$ go get git.mills.io/prologic/bitcask
 ```
 
-```#!go
+```go
 package main
 
-import "github.com/prologic/bitcask"
+import (
+	"log"
+	"git.mills.io/prologic/bitcask"
+)
 
 func main() {
     db, _ := bitcask.Open("/tmp/db")
     defer db.Close()
     db.Put([]byte("Hello"), []byte("World"))
     val, _ := db.Get([]byte("Hello"))
+    log.Printf(string(val))
 }
 ```
 
-See the [godoc](https://godoc.org/github.com/prologic/bitcask) for further
+See the [GoDoc](https://godoc.org/git.mills.io/prologic/bitcask) for further
 documentation and other examples.
 
 ## Usage (tool)
 
-```#!sh
+```sh
 $ bitcask -p /tmp/db set Hello World
 $ bitcask -p /tmp/db get Hello
 World
@@ -70,14 +133,14 @@ World
 
 There is also a builtin very  simple Redis-compatible server called `bitcaskd`:
 
-```#!sh
+```sh
 $ ./bitcaskd ./tmp
 INFO[0000] starting bitcaskd v0.0.7@146f777              bind=":6379" path=./tmp
 ```
 
 Example session:
 
-```#!sh
+```sh
 $ telnet localhost 6379
 Trying ::1...
 Connected to localhost.
@@ -102,21 +165,21 @@ Connection closed by foreign host.
 
 You can also use the [Bitcask Docker Image](https://cloud.docker.com/u/prologic/repository/docker/prologic/bitcask):
 
-```#!sh
+```sh
 $ docker pull prologic/bitcask
 $ docker run -d -p 6379:6379 prologic/bitcask
 ```
 
 ## Performance
 
-Benchmarks run on a 11" Macbook with a 1.4Ghz Intel Core i7:
+Benchmarks run on a 11" MacBook with a 1.4Ghz Intel Core i7:
 
-```#!sh
+```sh
 $ make bench
 ...
 goos: darwin
 goarch: amd64
-pkg: github.com/prologic/bitcask
+pkg: git.mills.io/prologic/bitcask
 
 BenchmarkGet/128B-4         	  316515	      3263 ns/op	  39.22 MB/s	     160 B/op	       1 allocs/op
 BenchmarkGet/256B-4         	  382551	      3204 ns/op	  79.90 MB/s	     288 B/op	       1 allocs/op
@@ -158,24 +221,39 @@ For 128B values:
 
 The full benchmark above shows linear performance as you increase key/value sizes.
 
-## Stargazers over time
-
-[![Stargazers over time](https://starcharts.herokuapp.com/prologic/bitcask.svg)](https://starcharts.herokuapp.com/prologic/bitcask)
-
 ## Support
 
 Support the ongoing development of Bitcask!
 
-**Sponser**
+**Sponsor**
 
 - Become a [Sponsor](https://www.patreon.com/prologic)
 
+## Stargazers over time
+
+[![Stargazers over time](https://starcharts.herokuapp.com/prologic/bitcask.svg)](https://starcharts.herokuapp.com/prologic/bitcask)
+
 ## Contributors
 
 Thank you to all those that have contributed to this project, battle-tested it, used it in their own projects or products, fixed bugs, improved performance and even fix tiny typos in documentation! Thank you and keep contributing!
 
-You can find an [AUTHORS](/AUTHORS) file where we keep a list of contributors to the project. If you contriibute a PR please consider adding your name there. There is also Github's own [Contributors](https://github.com/prologic/bitcask/graphs/contributors) statistics.
+You can find an [AUTHORS](/AUTHORS) file where we keep a list of contributors to the project. If you contribute a PR please consider adding your name there. There is also GitHub's own [Contributors](https://git.mills.io/prologic/bitcask/graphs/contributors) statistics.
+
+[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/0)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/0)
+[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/1)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/1)
+[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/2)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/2)
+[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/3)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/3)
+[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/4)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/4)
+[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/5)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/5)
+[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/6)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/6)
+[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/7)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/7)
+
+## Related Projects
+
+- [bitraft](https://git.mills.io/prologic/bitraft) -- A Distributed Key/Value store (_using Raft_) with a Redis compatible protocol.
+- [bitcaskfs](https://git.mills.io/prologic/bitcaskfs) -- A FUSE file system for mounting a Bitcask database.
+- [bitcask-bench](https://git.mills.io/prologic/bitcask-bench) -- A benchmarking tool comparing Bitcask and several other Go key/value libraries.
 
 ## License
 
-bitcask is licensed under the term of the [MIT License](https://github.com/prologic/bitcask/blob/master/LICENSE)
+bitcask is licensed under the term of the [MIT License](https://git.mills.io/prologic/bitcask/blob/master/LICENSE)

bitcask.go

@@ -2,20 +2,33 @@ package bitcask
 
 import (
 	"errors"
+	"fmt"
 	"hash/crc32"
 	"io"
 	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
+	"sort"
 	"sync"
+	"time"
 
 	"github.com/gofrs/flock"
 	art "github.com/plar/go-adaptive-radix-tree"
-	"github.com/prologic/bitcask/internal"
-	"github.com/prologic/bitcask/internal/config"
-	"github.com/prologic/bitcask/internal/data"
-	"github.com/prologic/bitcask/internal/index"
+	log "github.com/sirupsen/logrus"
+
+	"git.mills.io/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal/config"
+	"git.mills.io/prologic/bitcask/internal/data"
+	"git.mills.io/prologic/bitcask/internal/data/codec"
+	"git.mills.io/prologic/bitcask/internal/index"
+	"git.mills.io/prologic/bitcask/internal/metadata"
+	"git.mills.io/prologic/bitcask/scripts/migrations"
+)
+
+const (
+	lockfile     = "lock"
+	ttlIndexFile = "ttl_index"
 )
 
 var (
@@ -26,6 +39,13 @@ var (
 	// maximum allowed key size (configured with WithMaxKeySize).
 	ErrKeyTooLarge = errors.New("error: key too large")
 
+	// ErrKeyExpired is the error returned when a key is queried which has
+	// already expired (due to ttl)
+	ErrKeyExpired = errors.New("error: key expired")
+
+	// ErrEmptyKey is the error returned for a value with an empty key.
+	ErrEmptyKey = errors.New("error: empty key")
+
 	// ErrValueTooLarge is the error returned for a value that exceeds the
 	// maximum allowed value size (configured with WithMaxValueSize).
 	ErrValueTooLarge = errors.New("error: value too large")
@@ -37,23 +57,31 @@ var (
 	// ErrDatabaseLocked is the error returned if the database is locked
 	// (typically opened by another process)
 	ErrDatabaseLocked = errors.New("error: database locked")
+
+	ErrInvalidVersion = errors.New("error: invalid db version")
+
+	// ErrMergeInProgress is the error returned if merge is called when already a merge
+	// is in progress
+	ErrMergeInProgress = errors.New("error: merge already in progress")
 )
 
 // Bitcask is a struct that represents a on-disk LSM and WAL data structure
 // and in-memory hash of key/value pairs as per the Bitcask paper and seen
 // in the Riak database.
 type Bitcask struct {
-	mu sync.RWMutex
-
-	*flock.Flock
-
-	config    *config.Config
-	options   []Option
-	path      string
-	curr      data.Datafile
-	datafiles map[int]data.Datafile
-	trie      art.Tree
-	indexer   index.Indexer
+	mu         sync.RWMutex
+	flock      *flock.Flock
+	config     *config.Config
+	options    []Option
+	path       string
+	curr       data.Datafile
+	datafiles  map[int]data.Datafile
+	trie       art.Tree
+	indexer    index.Indexer
+	ttlIndexer index.Indexer
+	ttlIndex   art.Tree
+	metadata   *metadata.MetaData
+	isMerging  bool
 }
 
 // Stats is a struct returned by Stats() on an open Bitcask instance
@@ -82,12 +110,22 @@ func (b *Bitcask) Stats() (stats Stats, err error) {
 // Close() as this is the only way to cleanup the lock held by the open
 // database.
 func (b *Bitcask) Close() error {
+	b.mu.RLock()
 	defer func() {
-		b.Flock.Unlock()
-		os.Remove(b.Flock.Path())
+		b.mu.RUnlock()
+		b.flock.Unlock()
 	}()
 
-	if err := b.indexer.Save(b.trie, filepath.Join(b.path, "index")); err != nil {
+	return b.close()
+}
+
+func (b *Bitcask) close() error {
+	if err := b.saveIndexes(); err != nil {
+		return err
+	}
+
+	b.metadata.IndexUpToDate = true
+	if err := b.saveMetadata(); err != nil {
 		return err
 	}
 
@@ -102,96 +140,160 @@ func (b *Bitcask) Close() error {
 
 // Sync flushes all buffers to disk ensuring all data is written
 func (b *Bitcask) Sync() error {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+
+	if err := b.saveMetadata(); err != nil {
+		return err
+	}
+
 	return b.curr.Sync()
 }
 
-// Get retrieves the value of the given key. If the key is not found or an/I/O
-// error occurs a null byte slice is returned along with the error.
+// Get fetches value for a key
 func (b *Bitcask) Get(key []byte) ([]byte, error) {
-	var df data.Datafile
-
 	b.mu.RLock()
-	value, found := b.trie.Search(key)
-	if !found {
-		b.mu.RUnlock()
-		return nil, ErrKeyNotFound
-	}
-
-	item := value.(internal.Item)
-
-	if item.FileID == b.curr.FileID() {
-		df = b.curr
-	} else {
-		df = b.datafiles[item.FileID]
-	}
-
-	e, err := df.ReadAt(item.Offset, item.Size)
-	b.mu.RUnlock()
+	defer b.mu.RUnlock()
+	e, err := b.get(key)
 	if err != nil {
 		return nil, err
 	}
-
-	checksum := crc32.ChecksumIEEE(e.Value)
-	if checksum != e.Checksum {
-		return nil, ErrChecksumFailed
-	}
-
 	return e.Value, nil
 }
 
 // Has returns true if the key exists in the database, false otherwise.
 func (b *Bitcask) Has(key []byte) bool {
 	b.mu.RLock()
+	defer b.mu.RUnlock()
 	_, found := b.trie.Search(key)
-	b.mu.RUnlock()
+	if found {
+		return !b.isExpired(key)
+	}
 	return found
 }
 
 // Put stores the key and value in the database.
 func (b *Bitcask) Put(key, value []byte) error {
-	if uint32(len(key)) > b.config.MaxKeySize {
+	if len(key) == 0 {
+		return ErrEmptyKey
+	}
+	if b.config.MaxKeySize > 0 && uint32(len(key)) > b.config.MaxKeySize {
 		return ErrKeyTooLarge
 	}
-	if uint64(len(value)) > b.config.MaxValueSize {
+	if b.config.MaxValueSize > 0 && uint64(len(value)) > b.config.MaxValueSize {
 		return ErrValueTooLarge
 	}
 
 	b.mu.Lock()
+	defer b.mu.Unlock()
 	offset, n, err := b.put(key, value)
 	if err != nil {
-		b.mu.Unlock()
 		return err
 	}
 
 	if b.config.Sync {
 		if err := b.curr.Sync(); err != nil {
-			b.mu.Unlock()
 			return err
 		}
 	}
 
+	// in case of successful `put`, IndexUpToDate will be always be false
+	b.metadata.IndexUpToDate = false
+
+	if oldItem, found := b.trie.Search(key); found {
+		b.metadata.ReclaimableSpace += oldItem.(internal.Item).Size
+	}
+
 	item := internal.Item{FileID: b.curr.FileID(), Offset: offset, Size: n}
 	b.trie.Insert(key, item)
-	b.mu.Unlock()
 
 	return nil
 }
 
-// Delete deletes the named key. If the key doesn't exist or an I/O error
-// occurs the error is returned.
-func (b *Bitcask) Delete(key []byte) error {
+// PutWithTTL stores the key and value in the database with the given TTL
+func (b *Bitcask) PutWithTTL(key, value []byte, ttl time.Duration) error {
+	if len(key) == 0 {
+		return ErrEmptyKey
+	}
+	if b.config.MaxKeySize > 0 && uint32(len(key)) > b.config.MaxKeySize {
+		return ErrKeyTooLarge
+	}
+	if b.config.MaxValueSize > 0 && uint64(len(value)) > b.config.MaxValueSize {
+		return ErrValueTooLarge
+	}
+
+	expiry := time.Now().Add(ttl)
+
 	b.mu.Lock()
-	_, _, err := b.put(key, []byte{})
+	defer b.mu.Unlock()
+	offset, n, err := b.putWithExpiry(key, value, expiry)
 	if err != nil {
-		b.mu.Unlock()
 		return err
 	}
-	b.trie.Delete(key)
-	b.mu.Unlock()
+
+	if b.config.Sync {
+		if err := b.curr.Sync(); err != nil {
+			return err
+		}
+	}
+
+	// in case of successful `put`, IndexUpToDate will be always be false
+	b.metadata.IndexUpToDate = false
+
+	if oldItem, found := b.trie.Search(key); found {
+		b.metadata.ReclaimableSpace += oldItem.(internal.Item).Size
+	}
+
+	item := internal.Item{FileID: b.curr.FileID(), Offset: offset, Size: n}
+	b.trie.Insert(key, item)
+	b.ttlIndex.Insert(key, expiry)
 
 	return nil
 }
 
+// Delete deletes the named key.
+func (b *Bitcask) Delete(key []byte) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.delete(key)
+}
+
+// delete deletes the named key. If the key doesn't exist or an I/O error
+// occurs the error is returned.
+func (b *Bitcask) delete(key []byte) error {
+	_, _, err := b.put(key, []byte{})
+	if err != nil {
+		return err
+	}
+	if item, found := b.trie.Search(key); found {
+		b.metadata.ReclaimableSpace += item.(internal.Item).Size + codec.MetaInfoSize + int64(len(key))
+	}
+	b.trie.Delete(key)
+	b.ttlIndex.Delete(key)
+
+	return nil
+}
+
+// DeleteAll deletes all the keys. If an I/O error occurs the error is returned.
+func (b *Bitcask) DeleteAll() (err error) {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+
+	b.trie.ForEach(func(node art.Node) bool {
+		_, _, err = b.put(node.Key(), []byte{})
+		if err != nil {
+			return false
+		}
+		item, _ := b.trie.Search(node.Key())
+		b.metadata.ReclaimableSpace += item.(internal.Item).Size + codec.MetaInfoSize + int64(len(node.Key()))
+		return true
+	})
+	b.trie = art.New()
+	b.ttlIndex = art.New()
+
+	return
+}
+
 // Scan performs a prefix scan of keys matching the given prefix and calling
 // the function `f` with the keys found. If the function returns an error
 // no further keys are processed and the first error returned.
@@ -226,6 +328,9 @@ func (b *Bitcask) Keys() chan []byte {
 
 		for it := b.trie.Iterator(); it.HasNext(); {
 			node, _ := it.Next()
+			if b.isExpired(node.Key()) {
+				continue
+			}
 			ch <- node.Key()
 		}
 		close(ch)
@@ -234,6 +339,28 @@ func (b *Bitcask) Keys() chan []byte {
 	return ch
 }
 
+// RunGC deletes all expired keys
+func (b *Bitcask) RunGC() error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	return b.runGC()
+}
+
+// runGC deletes all keys that are expired
+// caller function should take care of the locking when calling this method
+func (b *Bitcask) runGC() (err error) {
+	b.ttlIndex.ForEach(func(node art.Node) (cont bool) {
+		if !b.isExpired(node.Key()) {
+			return true
+		}
+		if err = b.delete(node.Key()); err != nil {
+			return false
+		}
+		return true
+	})
+	return
+}
+
 // Fold iterates over all keys in the database calling the function `f` for
 // each key. If the function returns an error, no further keys are processed
 // and the error returned.
@@ -251,56 +378,178 @@ func (b *Bitcask) Fold(f func(key []byte) error) (err error) {
 	return
 }
 
-func (b *Bitcask) put(key, value []byte) (int64, int64, error) {
-	size := b.curr.Size()
-	if size >= int64(b.config.MaxDatafileSize) {
-		err := b.curr.Close()
-		if err != nil {
-			return -1, 0, err
-		}
+// get retrieves the value of the given key
+func (b *Bitcask) get(key []byte) (internal.Entry, error) {
+	var df data.Datafile
 
-		id := b.curr.FileID()
-
-		df, err := data.NewDatafile(b.path, id, true, b.config.MaxKeySize, b.config.MaxValueSize)
-		if err != nil {
-			return -1, 0, err
-		}
-
-		b.datafiles[id] = df
-
-		id = b.curr.FileID() + 1
-		curr, err := data.NewDatafile(b.path, id, false, b.config.MaxKeySize, b.config.MaxValueSize)
-		if err != nil {
-			return -1, 0, err
-		}
-		b.curr = curr
+	value, found := b.trie.Search(key)
+	if !found {
+		return internal.Entry{}, ErrKeyNotFound
+	}
+	if expired := b.isExpired(key); expired {
+		_ = b.delete(key) // we don't care if it doesnt succeed
+		return internal.Entry{}, ErrKeyExpired
 	}
 
-	e := internal.NewEntry(key, value)
-	return b.curr.Write(e)
+	item := value.(internal.Item)
+
+	if item.FileID == b.curr.FileID() {
+		df = b.curr
+	} else {
+		df = b.datafiles[item.FileID]
+	}
+
+	e, err := df.ReadAt(item.Offset, item.Size)
+	if err != nil {
+		return internal.Entry{}, err
+	}
+
+	checksum := crc32.ChecksumIEEE(e.Value)
+	if checksum != e.Checksum {
+		return internal.Entry{}, ErrChecksumFailed
+	}
+
+	return e, nil
 }
 
-func (b *Bitcask) reopen() error {
+func (b *Bitcask) maybeRotate() error {
+	size := b.curr.Size()
+	if size < int64(b.config.MaxDatafileSize) {
+		return nil
+	}
+
+	err := b.curr.Close()
+	if err != nil {
+		return err
+	}
+
+	id := b.curr.FileID()
+
+	df, err := data.NewDatafile(
+		b.path, id, true,
+		b.config.MaxKeySize,
+		b.config.MaxValueSize,
+		b.config.FileFileModeBeforeUmask,
+	)
+	if err != nil {
+		return err
+	}
+
+	b.datafiles[id] = df
+
+	id = b.curr.FileID() + 1
+	curr, err := data.NewDatafile(
+		b.path, id, false,
+		b.config.MaxKeySize,
+		b.config.MaxValueSize,
+		b.config.FileFileModeBeforeUmask,
+	)
+	if err != nil {
+		return err
+	}
+	b.curr = curr
+	err = b.saveIndexes()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// put inserts a new (key, value). Both key and value are valid inputs.
+func (b *Bitcask) put(key, value []byte) (int64, int64, error) {
+	if err := b.maybeRotate(); err != nil {
+		return -1, 0, fmt.Errorf("error rotating active datafile: %w", err)
+	}
+
+	return b.curr.Write(internal.NewEntry(key, value, nil))
+}
+
+// putWithExpiry inserts a new (key, value, expiry).
+// Both key and value are valid inputs.
+func (b *Bitcask) putWithExpiry(key, value []byte, expiry time.Time) (int64, int64, error) {
+	if err := b.maybeRotate(); err != nil {
+		return -1, 0, fmt.Errorf("error rotating active datafile: %w", err)
+	}
+
+	return b.curr.Write(internal.NewEntry(key, value, &expiry))
+}
+
+// closeCurrentFile closes current datafile and makes it read only.
+func (b *Bitcask) closeCurrentFile() error {
+	if err := b.curr.Close(); err != nil {
+		return err
+	}
+
+	id := b.curr.FileID()
+	df, err := data.NewDatafile(
+		b.path, id, true,
+		b.config.MaxKeySize,
+		b.config.MaxValueSize,
+		b.config.FileFileModeBeforeUmask,
+	)
+	if err != nil {
+		return err
+	}
+
+	b.datafiles[id] = df
+	return nil
+}
+
+// openNewWritableFile opens new datafile for writing data
+func (b *Bitcask) openNewWritableFile() error {
+	id := b.curr.FileID() + 1
+	curr, err := data.NewDatafile(
+		b.path, id, false,
+		b.config.MaxKeySize,
+		b.config.MaxValueSize,
+		b.config.FileFileModeBeforeUmask,
+	)
+	if err != nil {
+		return err
+	}
+	b.curr = curr
+	return nil
+}
+
+// Reopen closes and reopsns the database
+func (b *Bitcask) Reopen() error {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
-	datafiles, lastID, err := loadDatafiles(b.path, b.config.MaxKeySize, b.config.MaxValueSize)
+	return b.reopen()
+}
+
+// reopen reloads a bitcask object with index and datafiles
+// caller of this method should take care of locking
+func (b *Bitcask) reopen() error {
+	datafiles, lastID, err := loadDatafiles(
+		b.path,
+		b.config.MaxKeySize,
+		b.config.MaxValueSize,
+		b.config.FileFileModeBeforeUmask,
+	)
+	if err != nil {
+		return err
+	}
+	t, ttlIndex, err := loadIndexes(b, datafiles, lastID)
 	if err != nil {
 		return err
 	}
 
-	t, err := loadIndex(b.path, b.indexer, b.config.MaxKeySize, datafiles)
-	if err != nil {
-		return err
-	}
-
-	curr, err := data.NewDatafile(b.path, lastID, false, b.config.MaxKeySize, b.config.MaxValueSize)
+	curr, err := data.NewDatafile(
+		b.path, lastID, false,
+		b.config.MaxKeySize,
+		b.config.MaxValueSize,
+		b.config.FileFileModeBeforeUmask,
+	)
 	if err != nil {
 		return err
 	}
 
 	b.trie = t
 	b.curr = curr
+	b.ttlIndex = ttlIndex
 	b.datafiles = datafiles
 
 	return nil
@@ -310,6 +559,34 @@ func (b *Bitcask) reopen() error {
 // and deleted keys removes. Duplicate key/value pairs are also removed.
 // Call this function periodically to reclaim disk space.
 func (b *Bitcask) Merge() error {
+	b.mu.Lock()
+	if b.isMerging {
+		b.mu.Unlock()
+		return ErrMergeInProgress
+	}
+	b.isMerging = true
+	b.mu.Unlock()
+	defer func() {
+		b.isMerging = false
+	}()
+	b.mu.RLock()
+	err := b.closeCurrentFile()
+	if err != nil {
+		b.mu.RUnlock()
+		return err
+	}
+	filesToMerge := make([]int, 0, len(b.datafiles))
+	for k := range b.datafiles {
+		filesToMerge = append(filesToMerge, k)
+	}
+	err = b.openNewWritableFile()
+	if err != nil {
+		b.mu.RUnlock()
+		return err
+	}
+	b.mu.RUnlock()
+	sort.Ints(filesToMerge)
+
 	// Temporary merged database path
 	temp, err := ioutil.TempDir(b.path, "merge")
 	if err != nil {
@@ -318,7 +595,7 @@ func (b *Bitcask) Merge() error {
 	defer os.RemoveAll(temp)
 
 	// Create a merged database
-	mdb, err := Open(temp, b.options...)
+	mdb, err := Open(temp, withConfig(b.config))
 	if err != nil {
 		return err
 	}
@@ -327,13 +604,24 @@ func (b *Bitcask) Merge() error {
 	// Doing this automatically strips deleted keys and
 	// old key/value pairs
 	err = b.Fold(func(key []byte) error {
-		value, err := b.Get(key)
+		item, _ := b.trie.Search(key)
+		// if key was updated after start of merge operation, nothing to do
+		if item.(internal.Item).FileID > filesToMerge[len(filesToMerge)-1] {
+			return nil
+		}
+		e, err := b.get(key)
 		if err != nil {
 			return err
 		}
 
-		if err := mdb.Put(key, value); err != nil {
-			return err
+		if e.Expiry != nil {
+			if err := mdb.PutWithTTL(key, e.Value, time.Until(*e.Expiry)); err != nil {
+				return err
+			}
+		} else {
+			if err := mdb.Put(key, e.Value); err != nil {
+				return err
+			}
 		}
 
 		return nil
@@ -341,29 +629,36 @@ func (b *Bitcask) Merge() error {
 	if err != nil {
 		return err
 	}
-
-	err = mdb.Close()
-	if err != nil {
+	if err = mdb.Close(); err != nil {
+		return err
+	}
+	// no reads and writes till we reopen
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if err = b.close(); err != nil {
 		return err
 	}
 
-	// Close the database
-	err = b.Close()
-	if err != nil {
-		return err
-	}
-
-	// Remove all data files
+	// Remove data files
 	files, err := ioutil.ReadDir(b.path)
 	if err != nil {
 		return err
 	}
 	for _, file := range files {
-		if !file.IsDir() {
-			err := os.RemoveAll(path.Join([]string{b.path, file.Name()}...))
-			if err != nil {
-				return err
-			}
+		if file.IsDir() || file.Name() == lockfile {
+			continue
+		}
+		ids, err := internal.ParseIds([]string{file.Name()})
+		if err != nil {
+			return err
+		}
+		// if datafile was created after start of merge, skip
+		if len(ids) > 0 && ids[0] > filesToMerge[len(filesToMerge)-1] {
+			continue
+		}
+		err = os.RemoveAll(path.Join(b.path, file.Name()))
+		if err != nil {
+			return err
 		}
 	}
 
@@ -373,6 +668,10 @@ func (b *Bitcask) Merge() error {
 		return err
 	}
 	for _, file := range files {
+		// see #225
+		if file.Name() == lockfile {
+			continue
+		}
 		err := os.Rename(
 			path.Join([]string{mdb.path, file.Name()}...),
 			path.Join([]string{b.path, file.Name()}...),
@@ -381,6 +680,7 @@ func (b *Bitcask) Merge() error {
 			return err
 		}
 	}
+	b.metadata.ReclaimableSpace = 0
 
 	// And finally reopen the database
 	return b.reopen()
@@ -391,14 +691,11 @@ func (b *Bitcask) Merge() error {
 // configuration options as functions.
 func Open(path string, options ...Option) (*Bitcask, error) {
 	var (
-		cfg *config.Config
-		err error
+		cfg  *config.Config
+		err  error
+		meta *metadata.MetaData
 	)
 
-	if err := os.MkdirAll(path, 0755); err != nil {
-		return nil, err
-	}
-
 	configPath := filepath.Join(path, "config.json")
 	if internal.Exists(configPath) {
 		cfg, err = config.Load(configPath)
@@ -409,26 +706,41 @@ func Open(path string, options ...Option) (*Bitcask, error) {
 		cfg = newDefaultConfig()
 	}
 
-	bitcask := &Bitcask{
-		Flock:   flock.New(filepath.Join(path, "lock")),
-		config:  cfg,
-		options: options,
-		path:    path,
-		indexer: index.NewIndexer(),
+	if err := checkAndUpgrade(cfg, configPath); err != nil {
+		return nil, err
 	}
 
 	for _, opt := range options {
-		if err := opt(bitcask.config); err != nil {
+		if err := opt(cfg); err != nil {
 			return nil, err
 		}
 	}
 
-	locked, err := bitcask.Flock.TryLock()
+	if err := os.MkdirAll(path, cfg.DirFileModeBeforeUmask); err != nil {
+		return nil, err
+	}
+
+	meta, err = loadMetadata(path)
 	if err != nil {
 		return nil, err
 	}
 
-	if !locked {
+	bitcask := &Bitcask{
+		flock:      flock.New(filepath.Join(path, lockfile)),
+		config:     cfg,
+		options:    options,
+		path:       path,
+		indexer:    index.NewIndexer(),
+		ttlIndexer: index.NewTTLIndexer(),
+		metadata:   meta,
+	}
+
+	ok, err := bitcask.flock.TryLock()
+	if err != nil {
+		return nil, err
+	}
+
+	if !ok {
 		return nil, ErrDatabaseLocked
 	}
 
@@ -436,14 +748,83 @@ func Open(path string, options ...Option) (*Bitcask, error) {
 		return nil, err
 	}
 
-	if err := bitcask.reopen(); err != nil {
+	if cfg.AutoRecovery {
+		if err := data.CheckAndRecover(path, cfg); err != nil {
+			return nil, fmt.Errorf("recovering database: %s", err)
+		}
+	}
+	if err := bitcask.Reopen(); err != nil {
 		return nil, err
 	}
 
 	return bitcask, nil
 }
 
-func loadDatafiles(path string, maxKeySize uint32, maxValueSize uint64) (datafiles map[int]data.Datafile, lastID int, err error) {
+// checkAndUpgrade checks if DB upgrade is required
+// if yes, then applies version upgrade and saves updated config
+func checkAndUpgrade(cfg *config.Config, configPath string) error {
+	if cfg.DBVersion == CurrentDBVersion {
+		return nil
+	}
+	if cfg.DBVersion > CurrentDBVersion {
+		return ErrInvalidVersion
+	}
+	// for v0 to v1 upgrade, we need to append 8 null bytes after each encoded entry in datafiles
+	if cfg.DBVersion == uint32(0) && CurrentDBVersion == uint32(1) {
+		log.Warn("upgrading db version, might take some time....")
+		cfg.DBVersion = CurrentDBVersion
+		return migrations.ApplyV0ToV1(filepath.Dir(configPath), cfg.MaxDatafileSize)
+	}
+	return nil
+}
+
+// Backup copies db directory to given path
+// it creates path if it does not exist
+func (b *Bitcask) Backup(path string) error {
+	if !internal.Exists(path) {
+		if err := os.MkdirAll(path, b.config.DirFileModeBeforeUmask); err != nil {
+			return err
+		}
+	}
+	return internal.Copy(b.path, path, []string{lockfile})
+}
+
+// saveIndex saves index and ttl_index currently in RAM to disk
+func (b *Bitcask) saveIndexes() error {
+	tempIdx := "temp_index"
+	if err := b.indexer.Save(b.trie, filepath.Join(b.path, tempIdx)); err != nil {
+		return err
+	}
+	if err := os.Rename(filepath.Join(b.path, tempIdx), filepath.Join(b.path, "index")); err != nil {
+		return err
+	}
+	if err := b.ttlIndexer.Save(b.ttlIndex, filepath.Join(b.path, tempIdx)); err != nil {
+		return err
+	}
+	return os.Rename(filepath.Join(b.path, tempIdx), filepath.Join(b.path, ttlIndexFile))
+}
+
+// saveMetadata saves metadata into disk
+func (b *Bitcask) saveMetadata() error {
+	return b.metadata.Save(filepath.Join(b.path, "meta.json"), b.config.DirFileModeBeforeUmask)
+}
+
+// Reclaimable returns space that can be reclaimed
+func (b *Bitcask) Reclaimable() int64 {
+	return b.metadata.ReclaimableSpace
+}
+
+// isExpired returns true if a key has expired
+// it returns false if key does not exist in ttl index
+func (b *Bitcask) isExpired(key []byte) bool {
+	expiry, found := b.ttlIndex.Search(key)
+	if !found {
+		return false
+	}
+	return expiry.(time.Time).Before(time.Now().UTC())
+}
+
+func loadDatafiles(path string, maxKeySize uint32, maxValueSize uint64, fileModeBeforeUmask os.FileMode) (datafiles map[int]data.Datafile, lastID int, err error) {
 	fns, err := internal.GetDatafiles(path)
 	if err != nil {
 		return nil, 0, err
@@ -456,7 +837,12 @@ func loadDatafiles(path string, maxKeySize uint32, maxValueSize uint64) (datafil
 
 	datafiles = make(map[int]data.Datafile, len(ids))
 	for _, id := range ids {
-		datafiles[id], err = data.NewDatafile(path, id, true, maxKeySize, maxValueSize)
+		datafiles[id], err = data.NewDatafile(
+			path, id, true,
+			maxKeySize,
+			maxValueSize,
+			fileModeBeforeUmask,
+		)
 		if err != nil {
 			return
 		}
@@ -468,34 +854,79 @@ func loadDatafiles(path string, maxKeySize uint32, maxValueSize uint64) (datafil
 	return
 }
 
-func loadIndex(path string, indexer index.Indexer, maxKeySize uint32, datafiles map[int]data.Datafile) (art.Tree, error) {
-	t, found, err := indexer.Load(filepath.Join(path, "index"), maxKeySize)
-	if err != nil {
-		return nil, err
+func getSortedDatafiles(datafiles map[int]data.Datafile) []data.Datafile {
+	out := make([]data.Datafile, len(datafiles))
+	idx := 0
+	for _, df := range datafiles {
+		out[idx] = df
+		idx++
 	}
-	if !found {
-		for _, df := range datafiles {
-			var offset int64
-			for {
-				e, n, err := df.Read()
-				if err != nil {
-					if err == io.EOF {
-						break
-					}
-					return nil, err
-				}
+	sort.Slice(out, func(i, j int) bool {
+		return out[i].FileID() < out[j].FileID()
+	})
+	return out
+}
 
-				// Tombstone value  (deleted key)
-				if len(e.Value) == 0 {
-					t.Delete(e.Key)
-					offset += n
-					continue
-				}
-				item := internal.Item{FileID: df.FileID(), Offset: offset, Size: n}
-				t.Insert(e.Key, item)
-				offset += n
-			}
+// loadIndexes loads index from disk to memory. If index is not available or partially available (last bitcask process crashed)
+// then it iterates over last datafile and construct index
+// we construct ttl_index here also along with normal index
+func loadIndexes(b *Bitcask, datafiles map[int]data.Datafile, lastID int) (art.Tree, art.Tree, error) {
+	t, found, err := b.indexer.Load(filepath.Join(b.path, "index"), b.config.MaxKeySize)
+	if err != nil {
+		return nil, nil, err
+	}
+	ttlIndex, _, err := b.ttlIndexer.Load(filepath.Join(b.path, ttlIndexFile), b.config.MaxKeySize)
+	if err != nil {
+		return nil, nil, err
+	}
+	if found && b.metadata.IndexUpToDate {
+		return t, ttlIndex, nil
+	}
+	if found {
+		if err := loadIndexFromDatafile(t, ttlIndex, datafiles[lastID]); err != nil {
+			return nil, ttlIndex, err
+		}
+		return t, ttlIndex, nil
+	}
+	sortedDatafiles := getSortedDatafiles(datafiles)
+	for _, df := range sortedDatafiles {
+		if err := loadIndexFromDatafile(t, ttlIndex, df); err != nil {
+			return nil, ttlIndex, err
 		}
 	}
-	return t, nil
+	return t, ttlIndex, nil
+}
+
+func loadIndexFromDatafile(t art.Tree, ttlIndex art.Tree, df data.Datafile) error {
+	var offset int64
+	for {
+		e, n, err := df.Read()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return err
+		}
+		// Tombstone value  (deleted key)
+		if len(e.Value) == 0 {
+			t.Delete(e.Key)
+			offset += n
+			continue
+		}
+		item := internal.Item{FileID: df.FileID(), Offset: offset, Size: n}
+		t.Insert(e.Key, item)
+		if e.Expiry != nil {
+			ttlIndex.Insert(e.Key, *e.Expiry)
+		}
+		offset += n
+	}
+	return nil
+}
+
+func loadMetadata(path string) (*metadata.MetaData, error) {
+	if !internal.Exists(filepath.Join(path, "meta.json")) {
+		meta := new(metadata.MetaData)
+		return meta, nil
+	}
+	return metadata.Load(filepath.Join(path, "meta.json"))
 }

bitcask_test.go

@@ -6,18 +6,21 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"reflect"
 	"sort"
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
-	"github.com/prologic/bitcask/internal"
-	"github.com/prologic/bitcask/internal/config"
-	"github.com/prologic/bitcask/internal/mocks"
+	"git.mills.io/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal/config"
+	"git.mills.io/prologic/bitcask/internal/mocks"
 )
 
 var (
@@ -68,7 +71,7 @@ func TestAll(t *testing.T) {
 	})
 
 	t.Run("Put", func(t *testing.T) {
-		err = db.Put([]byte([]byte("foo")), []byte("bar"))
+		err = db.Put([]byte("foo"), []byte("bar"))
 		assert.NoError(err)
 	})
 
@@ -82,10 +85,40 @@ func TestAll(t *testing.T) {
 		assert.Equal(1, db.Len())
 	})
 
+	t.Run("PutWithTTL", func(t *testing.T) {
+		err = db.PutWithTTL([]byte("bar"), []byte("baz"), 0)
+		assert.NoError(err)
+	})
+
+	t.Run("GetExpiredKey", func(t *testing.T) {
+		time.Sleep(time.Millisecond)
+		_, err := db.Get([]byte("bar"))
+		assert.Error(err)
+		assert.Equal(ErrKeyExpired, err)
+	})
+
 	t.Run("Has", func(t *testing.T) {
 		assert.True(db.Has([]byte("foo")))
 	})
 
+	t.Run("HasWithExpired", func(t *testing.T) {
+		err = db.PutWithTTL([]byte("bar"), []byte("baz"), 0)
+		assert.NoError(err)
+		time.Sleep(time.Millisecond)
+		assert.False(db.Has([]byte("bar")))
+	})
+
+	t.Run("RunGC", func(t *testing.T) {
+		err = db.PutWithTTL([]byte("bar"), []byte("baz"), 0)
+		assert.NoError(err)
+		time.Sleep(time.Millisecond)
+		err = db.RunGC()
+		assert.NoError(err)
+		_, err := db.Get([]byte("bar"))
+		assert.Error(err)
+		assert.Equal(ErrKeyNotFound, err)
+	})
+
 	t.Run("Keys", func(t *testing.T) {
 		keys := make([][]byte, 0)
 		for key := range db.Keys() {
@@ -127,12 +160,125 @@ func TestAll(t *testing.T) {
 		assert.NoError(err)
 	})
 
+	t.Run("Backup", func(t *testing.T) {
+		path, err := ioutil.TempDir("", "backup")
+		defer os.RemoveAll(path)
+		assert.NoError(err)
+		err = db.Backup(filepath.Join(path, "db-backup"))
+		assert.NoError(err)
+	})
+
 	t.Run("Close", func(t *testing.T) {
 		err = db.Close()
 		assert.NoError(err)
 	})
 }
 
+func TestDeleteAll(t *testing.T) {
+	assert := assert.New(t)
+	testdir, _ := ioutil.TempDir("", "bitcask")
+	db, _ := Open(testdir)
+	_ = db.Put([]byte("foo"), []byte("foo"))
+	_ = db.Put([]byte("bar"), []byte("bar"))
+	_ = db.Put([]byte("baz"), []byte("baz"))
+	assert.Equal(3, db.Len())
+	err := db.DeleteAll()
+	assert.NoError(err)
+	assert.Equal(0, db.Len())
+	_, err = db.Get([]byte("foo"))
+	assert.Equal(ErrKeyNotFound, err)
+	_, err = db.Get([]byte("bar"))
+	assert.Equal(ErrKeyNotFound, err)
+	_, err = db.Get([]byte("baz"))
+	assert.Equal(ErrKeyNotFound, err)
+}
+
+func TestReopen1(t *testing.T) {
+	assert := assert.New(t)
+	for i := 0; i < 10; i++ {
+		testdir, _ := ioutil.TempDir("", "bitcask")
+		db, _ := Open(testdir, WithMaxDatafileSize(1))
+		_ = db.Put([]byte("foo"), []byte("bar"))
+		_ = db.Put([]byte("foo"), []byte("bar1"))
+		_ = db.Put([]byte("foo"), []byte("bar2"))
+		_ = db.Put([]byte("foo"), []byte("bar3"))
+		_ = db.Put([]byte("foo"), []byte("bar4"))
+		_ = db.Put([]byte("foo"), []byte("bar5"))
+		_ = db.Reopen()
+		val, _ := db.Get([]byte("foo"))
+		assert.Equal("bar5", string(val))
+	}
+}
+
+func TestReopen(t *testing.T) {
+	assert := assert.New(t)
+
+	testdir, err := ioutil.TempDir("", "bitcask")
+	assert.NoError(err)
+
+	t.Run("Reopen", func(t *testing.T) {
+		var (
+			db  *Bitcask
+			err error
+		)
+
+		t.Run("Open", func(t *testing.T) {
+			db, err = Open(testdir)
+			assert.NoError(err)
+		})
+
+		t.Run("Put", func(t *testing.T) {
+			err = db.Put([]byte("foo"), []byte("bar"))
+			assert.NoError(err)
+		})
+
+		t.Run("PutWithTTL", func(t *testing.T) {
+			err = db.PutWithTTL([]byte("bar"), []byte("baz"), 0)
+			assert.NoError(err)
+		})
+
+		t.Run("Get", func(t *testing.T) {
+			val, err := db.Get([]byte("foo"))
+			assert.NoError(err)
+			assert.Equal([]byte("bar"), val)
+		})
+
+		t.Run("Reopen", func(t *testing.T) {
+			err = db.Reopen()
+			assert.NoError(err)
+		})
+
+		t.Run("GetAfterReopen", func(t *testing.T) {
+			val, err := db.Get([]byte("foo"))
+			assert.NoError(err)
+			assert.Equal([]byte("bar"), val)
+		})
+
+		t.Run("PutAfterReopen", func(t *testing.T) {
+			err = db.Put([]byte("zzz"), []byte("foo"))
+			assert.NoError(err)
+		})
+
+		t.Run("GetAfterReopenAndPut", func(t *testing.T) {
+			val, err := db.Get([]byte("zzz"))
+			assert.NoError(err)
+			assert.Equal([]byte("foo"), val)
+		})
+
+		t.Run("GetExpiredKeyAfterReopen", func(t *testing.T) {
+			val, err := db.Get([]byte("bar"))
+			assert.Error(err)
+			assert.Equal(ErrKeyExpired, err)
+			assert.Nil(val)
+		})
+
+		t.Run("Close", func(t *testing.T) {
+			err = db.Close()
+			assert.NoError(err)
+		})
+	})
+}
+
 func TestDeletedKeys(t *testing.T) {
 	assert := assert.New(t)
 
@@ -204,6 +350,64 @@ func TestDeletedKeys(t *testing.T) {
 	})
 }
 
+func TestMetadata(t *testing.T) {
+	assert := assert.New(t)
+	testdir, err := ioutil.TempDir("", "bitcask")
+	assert.NoError(err)
+	defer os.RemoveAll(testdir)
+
+	db, err := Open(testdir)
+	assert.NoError(err)
+	err = db.Put([]byte("foo"), []byte("bar"))
+	assert.NoError(err)
+	err = db.Close()
+	assert.NoError(err)
+	db, err = Open(testdir)
+	assert.NoError(err)
+
+	t.Run("IndexUptoDateAfterCloseAndOpen", func(t *testing.T) {
+		assert.Equal(true, db.metadata.IndexUpToDate)
+	})
+	t.Run("IndexUptoDateAfterPut", func(t *testing.T) {
+		assert.NoError(db.Put([]byte("foo1"), []byte("bar1")))
+		assert.Equal(false, db.metadata.IndexUpToDate)
+	})
+	t.Run("Reclaimable", func(t *testing.T) {
+		assert.Equal(int64(0), db.Reclaimable())
+	})
+	t.Run("ReclaimableAfterNewPut", func(t *testing.T) {
+		assert.NoError(db.Put([]byte("hello"), []byte("world")))
+		assert.Equal(int64(0), db.Reclaimable())
+	})
+	t.Run("ReclaimableAfterRepeatedPut", func(t *testing.T) {
+		assert.NoError(db.Put([]byte("hello"), []byte("world")))
+		assert.Equal(int64(34), db.Reclaimable())
+	})
+	t.Run("ReclaimableAfterDelete", func(t *testing.T) {
+		assert.NoError(db.Delete([]byte("hello")))
+		assert.Equal(int64(97), db.Reclaimable())
+	})
+	t.Run("ReclaimableAfterNonExistingDelete", func(t *testing.T) {
+		assert.NoError(db.Delete([]byte("hello1")))
+		assert.Equal(int64(97), db.Reclaimable())
+	})
+	t.Run("ReclaimableAfterDeleteAll", func(t *testing.T) {
+		assert.NoError(db.DeleteAll())
+		assert.Equal(int64(214), db.Reclaimable())
+	})
+	t.Run("ReclaimableAfterMerge", func(t *testing.T) {
+		assert.NoError(db.Merge())
+		assert.Equal(int64(0), db.Reclaimable())
+	})
+	t.Run("IndexUptoDateAfterMerge", func(t *testing.T) {
+		assert.Equal(true, db.metadata.IndexUpToDate)
+	})
+	t.Run("ReclaimableAfterMergeAndDeleteAll", func(t *testing.T) {
+		assert.NoError(db.DeleteAll())
+		assert.Equal(int64(0), db.Reclaimable())
+	})
+}
+
 func TestConfigErrors(t *testing.T) {
 	assert := assert.New(t)
 
@@ -234,6 +438,119 @@ func TestConfigErrors(t *testing.T) {
 	})
 }
 
+func TestAutoRecovery(t *testing.T) {
+	withAutoRecovery := []bool{false, true}
+
+	for _, autoRecovery := range withAutoRecovery {
+		t.Run(fmt.Sprintf("%v", autoRecovery), func(t *testing.T) {
+			require := require.New(t)
+			testdir, err := ioutil.TempDir("", "bitcask")
+			require.NoError(err)
+			db, err := Open(testdir)
+			require.NoError(err)
+
+			// Insert 10 key-value pairs and verify all is ok.
+			makeKeyVal := func(i int) ([]byte, []byte) {
+				return []byte(fmt.Sprintf("foo%d", i)), []byte(fmt.Sprintf("bar%d", i))
+			}
+			n := 10
+			for i := 0; i < n; i++ {
+				key, val := makeKeyVal(i)
+				err = db.Put(key, val)
+				require.NoError(err)
+			}
+			for i := 0; i < n; i++ {
+				key, val := makeKeyVal(i)
+				rval, err := db.Get(key)
+				require.NoError(err)
+				require.Equal(val, rval)
+			}
+			err = db.Close()
+			require.NoError(err)
+
+			// Corrupt the last inserted key
+			f, err := os.OpenFile(path.Join(testdir, "000000000.data"), os.O_RDWR, 0755)
+			require.NoError(err)
+			fi, err := f.Stat()
+			require.NoError(err)
+			err = f.Truncate(fi.Size() - 1)
+			require.NoError(err)
+			err = f.Close()
+			require.NoError(err)
+
+			db, err = Open(testdir, WithAutoRecovery(autoRecovery))
+			t.Logf("err: %s", err)
+			require.NoError(err)
+			defer db.Close()
+			// Check that all values but the last are still intact.
+			for i := 0; i < 9; i++ {
+				key, val := makeKeyVal(i)
+				rval, err := db.Get(key)
+				require.NoError(err)
+				require.Equal(val, rval)
+			}
+			// Check the index has no more keys than non-corrupted ones.
+			// i.e: all but the last one.
+			numKeys := 0
+			for range db.Keys() {
+				numKeys++
+			}
+			if !autoRecovery {
+				// We are opening without autorepair, and thus are
+				// in a corrupted state. The index isn't coherent with
+				// the datafile.
+				require.Equal(n, numKeys)
+				return
+			}
+
+			require.Equal(n-1, numKeys, "The index should have n-1 keys")
+
+			// Double-check explicitly the corrupted one isn't here.
+			// This check is redundant considering the last two checks,
+			// but doesn't hurt.
+			corrKey, _ := makeKeyVal(9)
+			_, err = db.Get(corrKey)
+			require.Equal(ErrKeyNotFound, err)
+		})
+	}
+}
+
+func TestLoadIndexes(t *testing.T) {
+	assert := assert.New(t)
+	testdir, err1 := ioutil.TempDir("", "bitcask")
+	assert.NoError(err1)
+	defer os.RemoveAll(testdir)
+
+	var db *Bitcask
+	var err error
+
+	t.Run("Setup", func(t *testing.T) {
+		db, err = Open(testdir)
+		assert.NoError(err)
+		for i := 0; i < 5; i++ {
+			key := fmt.Sprintf("key%d", i)
+			val := fmt.Sprintf("val%d", i)
+			err := db.Put([]byte(key), []byte(val))
+			assert.NoError(err)
+		}
+		for i := 0; i < 5; i++ {
+			key := fmt.Sprintf("foo%d", i)
+			val := fmt.Sprintf("bar%d", i)
+			err := db.PutWithTTL([]byte(key), []byte(val), time.Duration(i)*time.Second)
+			assert.NoError(err)
+		}
+		err = db.Close()
+		assert.NoError(err)
+	})
+
+	t.Run("OpenAgain", func(t *testing.T) {
+		db, err = Open(testdir)
+		assert.NoError(err)
+		assert.Equal(10, db.trie.Size())
+		assert.Equal(5, db.ttlIndex.Size())
+	})
+}
+
 func TestReIndex(t *testing.T) {
 	assert := assert.New(t)
 
@@ -256,6 +573,16 @@ func TestReIndex(t *testing.T) {
 			assert.NoError(err)
 		})
 
+		t.Run("PutWithExpiry", func(t *testing.T) {
+			err = db.PutWithTTL([]byte("bar"), []byte("baz"), 0)
+			assert.NoError(err)
+		})
+
+		t.Run("PutWithLargeExpiry", func(t *testing.T) {
+			err = db.PutWithTTL([]byte("bar1"), []byte("baz1"), time.Hour)
+			assert.NoError(err)
+		})
+
 		t.Run("Get", func(t *testing.T) {
 			val, err := db.Get([]byte("foo"))
 			assert.NoError(err)
@@ -275,6 +602,8 @@ func TestReIndex(t *testing.T) {
 		t.Run("DeleteIndex", func(t *testing.T) {
 			err := os.Remove(filepath.Join(testdir, "index"))
 			assert.NoError(err)
+			err = os.Remove(filepath.Join(testdir, ttlIndexFile))
+			assert.NoError(err)
 		})
 	})
 
@@ -295,6 +624,16 @@ func TestReIndex(t *testing.T) {
 			assert.Equal([]byte("bar"), val)
 		})
 
+		t.Run("GetKeyWithExpiry", func(t *testing.T) {
+			val, err := db.Get([]byte("bar"))
+			assert.Error(err)
+			assert.Equal(ErrKeyExpired, err)
+			assert.Nil(val)
+			val, err = db.Get([]byte("bar1"))
+			assert.NoError(err)
+			assert.Equal([]byte("baz1"), val)
+		})
+
 		t.Run("Close", func(t *testing.T) {
 			err = db.Close()
 			assert.NoError(err)
@@ -396,6 +735,11 @@ func TestSync(t *testing.T) {
 		value := []byte("foobar")
 		err = db.Put(key, value)
 	})
+
+	t.Run("Put", func(t *testing.T) {
+		err = db.Put([]byte("hello"), []byte("world"))
+		assert.NoError(err)
+	})
 }
 
 func TestMaxKeySize(t *testing.T) {
@@ -523,7 +867,9 @@ func TestStatsError(t *testing.T) {
 			assert.Equal(stats.Datafiles, 0)
 			assert.Equal(stats.Keys, 1)
 		})
+	})
 
+	t.Run("Test", func(t *testing.T) {
 		t.Run("FabricatedDestruction", func(t *testing.T) {
 			// This would never happen in reality :D
 			// Or would it? :)
@@ -538,6 +884,162 @@ func TestStatsError(t *testing.T) {
 	})
 }
 
+func TestDirFileModeBeforeUmask(t *testing.T) {
+	assert := assert.New(t)
+
+	t.Run("Setup", func(t *testing.T) {
+		t.Run("Default DirFileModeBeforeUmask is 0700", func(t *testing.T) {
+			testdir, err := ioutil.TempDir("", "bitcask")
+			embeddedDir := filepath.Join(testdir, "cache")
+			assert.NoError(err)
+			defer os.RemoveAll(testdir)
+
+			defaultTestMode := os.FileMode(0700)
+
+			db, err := Open(embeddedDir)
+			assert.NoError(err)
+			defer db.Close()
+			err = filepath.Walk(testdir, func(path string, info os.FileInfo, err error) error {
+				// skip the root directory
+				if path == testdir {
+					return nil
+				}
+				if info.IsDir() {
+					// perms for directory on disk are filtered through defaultTestMode, AND umask of user running test.
+					// this means the mkdir calls can only FURTHER restrict permissions, not grant more (preventing escalatation).
+					// to make this test OS agnostic, we'll skip using golang.org/x/sys/unix, inferring umask via XOR and AND NOT.
+
+					// create anotherDir with allPerms - to infer umask
+					anotherDir := filepath.Join(testdir, "temp")
+					err := os.Mkdir(anotherDir, os.ModePerm)
+					assert.NoError(err)
+					defer os.RemoveAll(anotherDir)
+
+					anotherStat, err := os.Stat(anotherDir)
+					assert.NoError(err)
+
+					// infer umask from anotherDir
+					umask := os.ModePerm ^ (anotherStat.Mode() & os.ModePerm)
+
+					assert.Equal(info.Mode()&os.ModePerm, defaultTestMode&^umask)
+				}
+				return nil
+			})
+			assert.NoError(err)
+		})
+
+		t.Run("Dir FileModeBeforeUmask is set via options for all subdirectories", func(t *testing.T) {
+			testdir, err := ioutil.TempDir("", "bitcask")
+			embeddedDir := filepath.Join(testdir, "cache")
+			assert.NoError(err)
+			defer os.RemoveAll(testdir)
+
+			testMode := os.FileMode(0713)
+
+			db, err := Open(embeddedDir, WithDirFileModeBeforeUmask(testMode))
+			assert.NoError(err)
+			defer db.Close()
+			err = filepath.Walk(testdir, func(path string, info os.FileInfo, err error) error {
+				// skip the root directory
+				if path == testdir {
+					return nil
+				}
+				if info.IsDir() {
+					// create anotherDir with allPerms - to infer umask
+					anotherDir := filepath.Join(testdir, "temp")
+					err := os.Mkdir(anotherDir, os.ModePerm)
+					assert.NoError(err)
+					defer os.RemoveAll(anotherDir)
+
+					anotherStat, _ := os.Stat(anotherDir)
+
+					// infer umask from anotherDir
+					umask := os.ModePerm ^ (anotherStat.Mode() & os.ModePerm)
+
+					assert.Equal(info.Mode()&os.ModePerm, testMode&^umask)
+				}
+				return nil
+			})
+			assert.NoError(err)
+		})
+
+	})
+}
+
+func TestFileFileModeBeforeUmask(t *testing.T) {
+	assert := assert.New(t)
+
+	t.Run("Setup", func(t *testing.T) {
+		t.Run("Default File FileModeBeforeUmask is 0600", func(t *testing.T) {
+			testdir, err := ioutil.TempDir("", "bitcask")
+			assert.NoError(err)
+			defer os.RemoveAll(testdir)
+
+			defaultTestMode := os.FileMode(0600)
+
+			db, err := Open(testdir)
+			assert.NoError(err)
+			defer db.Close()
+			err = filepath.Walk(testdir, func(path string, info os.FileInfo, err error) error {
+				if !info.IsDir() {
+					// the lock file is set within Flock, so ignore it
+					if filepath.Base(path) == "lock" {
+						return nil
+					}
+					// create aFile with allPerms - to infer umask
+					aFilePath := filepath.Join(testdir, "temp")
+					_, err := os.OpenFile(aFilePath, os.O_CREATE, os.ModePerm)
+					assert.NoError(err)
+					defer os.RemoveAll(aFilePath)
+
+					fileStat, _ := os.Stat(aFilePath)
+
+					// infer umask from anotherDir
+					umask := os.ModePerm ^ (fileStat.Mode() & os.ModePerm)
+
+					assert.Equal(info.Mode()&os.ModePerm, defaultTestMode&^umask)
+				}
+				return nil
+			})
+			assert.NoError(err)
+		})
+
+		t.Run("File FileModeBeforeUmask is set via options for all files", func(t *testing.T) {
+			testdir, err := ioutil.TempDir("", "bitcask")
+			assert.NoError(err)
+			defer os.RemoveAll(testdir)
+
+			testMode := os.FileMode(0673)
+
+			db, err := Open(testdir, WithFileFileModeBeforeUmask(testMode))
+			assert.NoError(err)
+			defer db.Close()
+			err = filepath.Walk(testdir, func(path string, info os.FileInfo, err error) error {
+				if !info.IsDir() {
+					// the lock file is set within Flock, so ignore it
+					if filepath.Base(path) == "lock" {
+						return nil
+					}
+					// create aFile with allPerms - to infer umask
+					aFilePath := filepath.Join(testdir, "temp")
+					_, err := os.OpenFile(aFilePath, os.O_CREATE, os.ModePerm)
+					assert.NoError(err)
+					defer os.RemoveAll(aFilePath)
+
+					fileStat, _ := os.Stat(aFilePath)
+
+					// infer umask from anotherDir
+					umask := os.ModePerm ^ (fileStat.Mode() & os.ModePerm)
+
+					assert.Equal(info.Mode()&os.ModePerm, testMode&^umask)
+				}
+				return nil
+			})
+			assert.NoError(err)
+		})
+	})
+}
+
 func TestMaxDatafileSize(t *testing.T) {
 	var (
 		db  *Bitcask
@@ -639,7 +1141,7 @@ func TestMerge(t *testing.T) {
 
 		s3, err := db.Stats()
 		assert.NoError(err)
-		assert.Equal(1, s3.Datafiles)
+		assert.Equal(2, s3.Datafiles)
 		assert.Equal(1, s3.Keys)
 		assert.True(s3.Size > s1.Size)
 		assert.True(s3.Size < s2.Size)
@@ -672,7 +1174,7 @@ func TestGetErrors(t *testing.T) {
 
 		mockDatafile := new(mocks.Datafile)
 		mockDatafile.On("FileID").Return(0)
-		mockDatafile.On("ReadAt", int64(0), int64(22)).Return(
+		mockDatafile.On("ReadAt", int64(0), int64(30)).Return(
 			internal.Entry{},
 			ErrMockError,
 		)
@@ -688,7 +1190,7 @@ func TestGetErrors(t *testing.T) {
 		assert.NoError(err)
 		defer os.RemoveAll(testdir)
 
-		db, err := Open(testdir, WithMaxDatafileSize(32))
+		db, err := Open(testdir, WithMaxDatafileSize(40))
 		assert.NoError(err)
 
 		err = db.Put([]byte("foo"), []byte("bar"))
@@ -696,7 +1198,7 @@ func TestGetErrors(t *testing.T) {
 
 		mockDatafile := new(mocks.Datafile)
 		mockDatafile.On("FileID").Return(0)
-		mockDatafile.On("ReadAt", int64(0), int64(22)).Return(
+		mockDatafile.On("ReadAt", int64(0), int64(30)).Return(
 			internal.Entry{
 				Checksum: 0x0,
 				Key:      []byte("foo"),
@@ -714,6 +1216,24 @@ func TestGetErrors(t *testing.T) {
 
 }
 
+func TestPutBorderCases(t *testing.T) {
+	assert := assert.New(t)
+
+	t.Run("EmptyValue", func(t *testing.T) {
+		testdir, err := ioutil.TempDir("", "bitcask")
+		assert.NoError(err)
+
+		db, err := Open(testdir)
+		assert.NoError(err)
+
+		err = db.Put([]byte("alice"), nil)
+		assert.NoError(err)
+		z, err := db.Get([]byte("alice"))
+		assert.NoError(err)
+		assert.Empty(z)
+	})
+}
+
 func TestPutErrors(t *testing.T) {
 	assert := assert.New(t)
 
@@ -766,6 +1286,18 @@ func TestPutErrors(t *testing.T) {
 		assert.Error(err)
 		assert.Equal(ErrMockError, err)
 	})
+
+	t.Run("EmptyKey", func(t *testing.T) {
+		testdir, err := ioutil.TempDir("", "bitcask")
+		assert.NoError(err)
+
+		db, err := Open(testdir)
+		assert.NoError(err)
+		err = db.Put(nil, []byte("hello"))
+		assert.Equal(ErrEmptyKey, err)
+
+	})
+
 }
 
 func TestOpenErrors(t *testing.T) {
@@ -833,7 +1365,7 @@ func TestCloseErrors(t *testing.T) {
 		assert.NoError(err)
 
 		mockIndexer := new(mocks.Indexer)
-		mockIndexer.On("Save", db.trie, filepath.Join(db.path, "index")).Return(ErrMockError)
+		mockIndexer.On("Save", db.trie, filepath.Join(db.path, "temp_index")).Return(ErrMockError)
 		db.indexer = mockIndexer
 
 		err = db.Close()
@@ -939,18 +1471,19 @@ func TestMergeErrors(t *testing.T) {
 		assert.NoError(err)
 		defer os.RemoveAll(testdir)
 
-		db, err := Open(testdir)
+		db, err := Open(testdir, WithMaxDatafileSize(22))
 		assert.NoError(err)
 
 		assert.NoError(db.Put([]byte("foo"), []byte("bar")))
+		assert.NoError(db.Put([]byte("bar"), []byte("baz")))
 
 		mockDatafile := new(mocks.Datafile)
-		mockDatafile.On("FileID").Return(0)
-		mockDatafile.On("ReadAt", int64(0), int64(22)).Return(
+		mockDatafile.On("Close").Return(nil)
+		mockDatafile.On("ReadAt", int64(0), int64(30)).Return(
 			internal.Entry{},
 			ErrMockError,
 		)
-		db.curr = mockDatafile
+		db.datafiles[0] = mockDatafile
 
 		err = db.Merge()
 		assert.Error(err)
@@ -999,11 +1532,11 @@ func TestConcurrent(t *testing.T) {
 			}
 
 			wg := &sync.WaitGroup{}
+			wg.Add(3)
 
 			go f(wg, 2)
 			go f(wg, 3)
 			go f(wg, 5)
-			wg.Add(3)
 
 			wg.Wait()
 		})
@@ -1021,15 +1554,100 @@ func TestConcurrent(t *testing.T) {
 			}
 
 			wg := &sync.WaitGroup{}
-
-			go f(wg, 100)
-			go f(wg, 100)
-			go f(wg, 100)
 			wg.Add(3)
+			go f(wg, 100)
+			go f(wg, 100)
+			go f(wg, 100)
 
 			wg.Wait()
 		})
 
+		// Test concurrent Put() with concurrent Scan()
+		t.Run("PutScan", func(t *testing.T) {
+			doPut := func(wg *sync.WaitGroup, x int) {
+				defer func() {
+					wg.Done()
+				}()
+				for i := 0; i <= 100; i++ {
+					if i%x == 0 {
+						key := []byte(fmt.Sprintf("k%d", i))
+						value := []byte(fmt.Sprintf("v%d", i))
+						err := db.Put(key, value)
+						assert.NoError(err)
+					}
+				}
+			}
+
+			doScan := func(wg *sync.WaitGroup, x int) {
+				defer func() {
+					wg.Done()
+				}()
+				for i := 0; i <= 100; i++ {
+					if i%x == 0 {
+						err := db.Scan([]byte("k"), func(key []byte) error {
+							return nil
+						})
+						assert.NoError(err)
+					}
+				}
+			}
+
+			wg := &sync.WaitGroup{}
+			wg.Add(6)
+
+			go doPut(wg, 2)
+			go doPut(wg, 3)
+			go doPut(wg, 5)
+			go doScan(wg, 1)
+			go doScan(wg, 2)
+			go doScan(wg, 4)
+
+			wg.Wait()
+		})
+
+		// XXX: This has data races
+		/* Test concurrent Scan() with concurrent Merge()
+		t.Run("ScanMerge", func(t *testing.T) {
+			doScan := func(wg *sync.WaitGroup, x int) {
+				defer func() {
+					wg.Done()
+				}()
+				for i := 0; i <= 100; i++ {
+					if i%x == 0 {
+						err := db.Scan([]byte("k"), func(key []byte) error {
+							return nil
+						})
+						assert.NoError(err)
+					}
+				}
+			}
+
+			doMerge := func(wg *sync.WaitGroup, x int) {
+				defer func() {
+					wg.Done()
+				}()
+				for i := 0; i <= 100; i++ {
+					if i%x == 0 {
+						err := db.Merge()
+						assert.NoError(err)
+					}
+				}
+			}
+
+			wg := &sync.WaitGroup{}
+			wg.Add(6)
+
+			go doScan(wg, 2)
+			go doScan(wg, 3)
+			go doScan(wg, 5)
+			go doMerge(wg, 1)
+			go doMerge(wg, 2)
+			go doMerge(wg, 4)
+
+			wg.Wait()
+		})
+		*/
+
 		t.Run("Close", func(t *testing.T) {
 			err = db.Close()
 			assert.NoError(err)
@@ -1109,7 +1727,27 @@ func TestLocking(t *testing.T) {
 
 	_, err = Open(testdir)
 	assert.Error(err)
-	assert.Equal(ErrDatabaseLocked, err)
+}
+
+func TestLockingAfterMerge(t *testing.T) {
+	assert := assert.New(t)
+
+	testdir, err := ioutil.TempDir("", "bitcask")
+	assert.NoError(err)
+
+	db, err := Open(testdir)
+	assert.NoError(err)
+	defer db.Close()
+
+	_, err = Open(testdir)
+	assert.Error(err)
+
+	err = db.Merge()
+	assert.NoError(err)
+
+	// This should still error.
+	_, err = Open(testdir)
+	assert.Error(err)
 }
 
 type benchmarkTestCase struct {

cmd/bitcask/del.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var delCmd = &cobra.Command{

cmd/bitcask/export.go

@@ -11,7 +11,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var errNotAllDataWritten = errors.New("error: not all data written")

cmd/bitcask/get.go

@@ -8,7 +8,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var getCmd = &cobra.Command{

cmd/bitcask/import.go

@@ -11,7 +11,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var importCmd = &cobra.Command{

cmd/bitcask/initdb.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var initdbCmd = &cobra.Command{

cmd/bitcask/keys.go

@@ -8,7 +8,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var keysCmd = &cobra.Command{

cmd/bitcask/merge.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var mergeCmd = &cobra.Command{

cmd/bitcask/put.go

@@ -10,7 +10,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var putCmd = &cobra.Command{

cmd/bitcask/recover.go

@@ -6,11 +6,11 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/prologic/bitcask"
-	"github.com/prologic/bitcask/internal"
-	"github.com/prologic/bitcask/internal/config"
-	"github.com/prologic/bitcask/internal/data/codec"
-	"github.com/prologic/bitcask/internal/index"
+	"git.mills.io/prologic/bitcask"
+	"git.mills.io/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal/config"
+	"git.mills.io/prologic/bitcask/internal/data/codec"
+	"git.mills.io/prologic/bitcask/internal/index"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"

cmd/bitcask/root.go

@@ -8,7 +8,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 )
 
 // RootCmd represents the base command when called without any subcommands

cmd/bitcask/scan.go

@@ -8,7 +8,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var scanCmd = &cobra.Command{

cmd/bitcask/stats.go

@@ -9,7 +9,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var statsCmd = &cobra.Command{

cmd/bitcaskd/main.go

@@ -7,7 +7,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	flag "github.com/spf13/pflag"
 
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 )
 
 var (
@@ -55,5 +55,7 @@ func main() {
 		os.Exit(2)
 	}
 
-	log.Fatal(server.Run())
+	if err = server.Run(); err != nil {
+		log.Fatal(err)
+	}
 }

cmd/bitcaskd/server.go

@@ -1,13 +1,18 @@
 package main
 
 import (
+	"encoding/binary"
 	"fmt"
+	"os"
+	"os/signal"
 	"strings"
+	"syscall"
+	"time"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/tidwall/redcon"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 type server struct {
@@ -29,17 +34,37 @@ func newServer(bind, dbpath string) (*server, error) {
 }
 
 func (s *server) handleSet(cmd redcon.Command, conn redcon.Conn) {
-	if len(cmd.Args) != 3 {
+	if len(cmd.Args) != 3 && len(cmd.Args) != 4 {
 		conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
 		return
 	}
+
+	var ttl *time.Duration
+
 	key := cmd.Args[1]
 	value := cmd.Args[2]
-	if err := s.db.Put(key, value); err != nil {
-		conn.WriteString(fmt.Sprintf("ERR: %s", err))
-	} else {
-		conn.WriteString("OK")
+
+	if len(cmd.Args) == 4 {
+		val, n := binary.Varint(cmd.Args[3])
+		if n <= 0 {
+			conn.WriteError("ERR error parsing ttl")
+			return
+		}
+		d := time.Duration(val) * time.Millisecond
+		ttl = &d
 	}
+
+	if ttl != nil {
+		if err := s.db.PutWithTTL(key, value, *ttl); err != nil {
+			conn.WriteString(fmt.Sprintf("ERR: %s", err))
+		}
+	} else {
+		if err := s.db.Put(key, value); err != nil {
+			conn.WriteString(fmt.Sprintf("ERR: %s", err))
+		}
+	}
+
+	conn.WriteString("OK")
 }
 
 func (s *server) handleGet(cmd redcon.Command, conn redcon.Conn) {
@@ -47,7 +72,9 @@ func (s *server) handleGet(cmd redcon.Command, conn redcon.Conn) {
 		conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
 		return
 	}
+
 	key := cmd.Args[1]
+
 	value, err := s.db.Get(key)
 	if err != nil {
 		conn.WriteNull()
@@ -59,7 +86,7 @@ func (s *server) handleGet(cmd redcon.Command, conn redcon.Conn) {
 func (s *server) handleKeys(cmd redcon.Command, conn redcon.Conn) {
 	conn.WriteArray(s.db.Len())
 	for key := range s.db.Keys() {
-		conn.WriteBulk([]byte(key))
+		conn.WriteBulk(key)
 	}
 }
 
@@ -68,7 +95,9 @@ func (s *server) handleExists(cmd redcon.Command, conn redcon.Conn) {
 		conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
 		return
 	}
+
 	key := cmd.Args[1]
+
 	if s.db.Has(key) {
 		conn.WriteInt(1)
 	} else {
@@ -81,7 +110,9 @@ func (s *server) handleDel(cmd redcon.Command, conn redcon.Conn) {
 		conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
 		return
 	}
+
 	key := cmd.Args[1]
+
 	if err := s.db.Delete(key); err != nil {
 		conn.WriteInt(0)
 	} else {
@@ -89,8 +120,13 @@ func (s *server) handleDel(cmd redcon.Command, conn redcon.Conn) {
 	}
 }
 
+func (s *server) Shutdown() (err error) {
+	err = s.db.Close()
+	return
+}
+
 func (s *server) Run() (err error) {
-	err = redcon.ListenAndServe(s.bind,
+	redServer := redcon.NewServerNetwork("tcp", s.bind,
 		func(conn redcon.Conn, cmd redcon.Command) {
 			switch strings.ToLower(string(cmd.Args[0])) {
 			case "ping":
@@ -118,5 +154,17 @@ func (s *server) Run() (err error) {
 		func(conn redcon.Conn, err error) {
 		},
 	)
+
+	go func() {
+		signals := make(chan os.Signal, 1)
+		signal.Notify(signals, os.Interrupt, syscall.SIGTERM)
+		s := <-signals
+		log.Infof("Shutdown server on signal %s", s)
+		redServer.Close()
+	}()
+
+	if err := redServer.ListenAndServe(); err == nil {
+		return s.Shutdown()
+	}
 	return
 }

go.mod

@@ -1,23 +1,17 @@
-module github.com/prologic/bitcask
+module git.mills.io/prologic/bitcask
 
 go 1.13
 
 require (
-	github.com/gofrs/flock v0.7.1
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
-	github.com/magiconair/properties v1.8.1 // indirect
-	github.com/pelletier/go-toml v1.4.0 // indirect
-	github.com/pkg/errors v0.8.1
-	github.com/plar/go-adaptive-radix-tree v1.0.1
-	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/afero v1.2.2 // indirect
-	github.com/spf13/cobra v0.0.5
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/pflag v1.0.3
-	github.com/spf13/viper v1.4.0
-	github.com/stretchr/testify v1.3.0
-	github.com/tidwall/redcon v1.0.0
-	golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56
-	golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa // indirect
-	golang.org/x/text v0.3.2 // indirect
+	github.com/gofrs/flock v0.8.0
+	github.com/pkg/errors v0.9.1
+	github.com/plar/go-adaptive-radix-tree v1.0.4
+	github.com/sirupsen/logrus v1.8.1
+	github.com/spf13/cobra v0.0.7
+	github.com/spf13/pflag v1.0.5
+	github.com/spf13/viper v1.8.1
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/stretchr/testify v1.7.0
+	github.com/tidwall/redcon v1.4.1
+	golang.org/x/exp v0.0.0-20200228211341-fcea875c7e85
 )

go.sum

@@ -1,191 +1,688 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
+cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
+cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
+cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
+cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gofrs/flock v0.7.1 h1:DP+LD/t0njgoPBvT5MJLeliUIVQR03hiKR6vezdwHlc=
-github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gofrs/flock v0.8.0 h1:MSdYClljsF3PbENUUEx85nkWfJSGfzYI9yEBZOJz6CY=
+github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
-github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
+github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
+github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.4.0 h1:u3Z1r+oOXJIkxqw34zVhyPgjBsm6X2wn21NWs/HfSeg=
-github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo=
+github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ=
+github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/plar/go-adaptive-radix-tree v1.0.1 h1:J+2qrXaKWLACw59s8SlTVYYxWjlUr/BlCsfkAzn96/0=
-github.com/plar/go-adaptive-radix-tree v1.0.1/go.mod h1:Ot8d28EII3i7Lv4PSvBlF8ejiD/CtRYDuPsySJbSaK8=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
+github.com/plar/go-adaptive-radix-tree v1.0.4 h1:Ucd8R6RH2E7RW8ZtDKrsWyOD3paG2qqJO0I20WQ8oWQ=
+github.com/plar/go-adaptive-radix-tree v1.0.4/go.mod h1:Ot8d28EII3i7Lv4PSvBlF8ejiD/CtRYDuPsySJbSaK8=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
-github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY=
+github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
-github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
-github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
+github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.7 h1:FfTH+vuMXOas8jmfb5/M7dzEYx7LpcLb7a0LPe34uOU=
+github.com/spf13/cobra v0.0.7/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
-github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44=
+github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/tidwall/redcon v1.0.0 h1:D4AzzJ81Afeh144fgnj5H0aSVPBBJ5RI9Rzj0zThU+E=
-github.com/tidwall/redcon v1.0.0/go.mod h1:bdYBm4rlcWpst2XMwKVzWDF9CoUxEbUmM7CQrKeOZas=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/tidwall/btree v0.4.2 h1:aLwwJlG+InuFzdAPuBf9YCAR1LvSQ9zhC5aorFPlIPs=
+github.com/tidwall/btree v0.4.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
+github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
+github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/redcon v1.4.1 h1:oupK+lM1FeSGNhhZn85KvofEpboQReM1eIKNWTmD3K8=
+github.com/tidwall/redcon v1.4.1/go.mod h1:XwNPFbJ4ShWNNSA2Jazhbdje6jegTCcwFR6mfaADvHA=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
-github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
+go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56 h1:estk1glOnSVeJ9tdEZZc5mAMDZk5lNJNyJ6DvrBkTEU=
-golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20200228211341-fcea875c7e85 h1:jqhIzSw5SQNkbu5hOGpgMHhkfXxrbsLJdkIRcX19gCY=
+golang.org/x/exp v0.0.0-20200228211341-fcea875c7e85/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
-golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa h1:KIDDMLT1O0Nr7TSxp8xM5tJcdn8tgyAONntO829og1M=
-golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846 h1:0oJP+9s5Z3MT6dym56c4f7nVeujVpL1QyD2Vp/bTql0=
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
+google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
+google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
+google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
+google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
+google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
+google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
+google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=
+gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

internal/config/config.go

@@ -8,10 +8,14 @@ import (
 
 // Config contains the bitcask configuration parameters
 type Config struct {
-	MaxDatafileSize int    `json:"max_datafile_size"`
-	MaxKeySize      uint32 `json:"max_key_size"`
-	MaxValueSize    uint64 `json:"max_value_size"`
-	Sync            bool   `json:"sync"`
+	MaxDatafileSize         int    `json:"max_datafile_size"`
+	MaxKeySize              uint32 `json:"max_key_size"`
+	MaxValueSize            uint64 `json:"max_value_size"`
+	Sync                    bool   `json:"sync"`
+	AutoRecovery            bool   `json:"autorecovery"`
+	DBVersion               uint32 `json:"db_version"`
+	DirFileModeBeforeUmask  os.FileMode
+	FileFileModeBeforeUmask os.FileMode
 }
 
 // Load loads a configuration from the given path
@@ -32,23 +36,16 @@ func Load(path string) (*Config, error) {
 
 // Save saves the configuration to the provided path
 func (c *Config) Save(path string) error {
-	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
-	if err != nil {
-		return err
-	}
 
 	data, err := json.Marshal(c)
 	if err != nil {
 		return err
 	}
 
-	if _, err = f.Write(data); err != nil {
+	err = ioutil.WriteFile(path, data, c.FileFileModeBeforeUmask)
+	if err != nil {
 		return err
 	}
 
-	if err = f.Sync(); err != nil {
-		return err
-	}
-
-	return f.Close()
+	return nil
 }

internal/data/codec/decoder.go

@@ -3,9 +3,10 @@ package codec
 import (
 	"encoding/binary"
 	"io"
+	"time"
 
 	"github.com/pkg/errors"
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 )
 
 var (
@@ -49,13 +50,13 @@ func (d *Decoder) Decode(v *internal.Entry) (int64, error) {
 		return 0, err
 	}
 
-	buf := make([]byte, uint64(actualKeySize)+actualValueSize+checksumSize)
+	buf := make([]byte, uint64(actualKeySize)+actualValueSize+checksumSize+ttlSize)
 	if _, err = io.ReadFull(d.r, buf); err != nil {
 		return 0, errTruncatedData
 	}
 
 	decodeWithoutPrefix(buf, actualKeySize, v)
-	return int64(keySize + valueSize + uint64(actualKeySize) + actualValueSize + checksumSize), nil
+	return int64(keySize + valueSize + uint64(actualKeySize) + actualValueSize + checksumSize + ttlSize), nil
 }
 
 // DecodeEntry decodes a serialized entry
@@ -74,7 +75,7 @@ func getKeyValueSizes(buf []byte, maxKeySize uint32, maxValueSize uint64) (uint3
 	actualKeySize := binary.BigEndian.Uint32(buf[:keySize])
 	actualValueSize := binary.BigEndian.Uint64(buf[keySize:])
 
-	if actualKeySize > maxKeySize || actualValueSize > maxValueSize || actualKeySize == 0 {
+	if (maxKeySize > 0 && actualKeySize > maxKeySize) || (maxValueSize > 0 && actualValueSize > maxValueSize) || actualKeySize == 0 {
 
 		return 0, 0, errInvalidKeyOrValueSize
 	}
@@ -84,8 +85,18 @@ func getKeyValueSizes(buf []byte, maxKeySize uint32, maxValueSize uint64) (uint3
 
 func decodeWithoutPrefix(buf []byte, valueOffset uint32, v *internal.Entry) {
 	v.Key = buf[:valueOffset]
-	v.Value = buf[valueOffset : len(buf)-checksumSize]
-	v.Checksum = binary.BigEndian.Uint32(buf[len(buf)-checksumSize:])
+	v.Value = buf[valueOffset : len(buf)-checksumSize-ttlSize]
+	v.Checksum = binary.BigEndian.Uint32(buf[len(buf)-checksumSize-ttlSize : len(buf)-ttlSize])
+	v.Expiry = getKeyExpiry(buf)
+}
+
+func getKeyExpiry(buf []byte) *time.Time {
+	expiry := binary.BigEndian.Uint64(buf[len(buf)-ttlSize:])
+	if expiry == uint64(0) {
+		return nil
+	}
+	t := time.Unix(int64(expiry), 0).UTC()
+	return &t
 }
 
 // IsCorruptedData indicates if the error correspondes to possible data corruption

internal/data/codec/decoder_test.go

@@ -5,8 +5,9 @@ import (
 	"encoding/binary"
 	"io"
 	"testing"
+	"time"
 
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -107,3 +108,23 @@ func TestTruncatedData(t *testing.T) {
 		})
 	}
 }
+
+func TestDecodeWithoutPrefix(t *testing.T) {
+	assert := assert.New(t)
+	e := internal.Entry{}
+	buf := []byte{0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 7, 109, 121, 107, 101, 121, 109, 121, 118, 97, 108, 117, 101, 0, 6, 81, 189, 0, 0, 0, 0, 95, 117, 28, 0}
+	valueOffset := uint32(5)
+	mockTime := time.Date(2020, 10, 1, 0, 0, 0, 0, time.UTC)
+	expectedEntry := internal.Entry{
+		Key:      []byte("mykey"),
+		Value:    []byte("myvalue"),
+		Checksum: 414141,
+		Expiry:   &mockTime,
+	}
+	decodeWithoutPrefix(buf[keySize+valueSize:], valueOffset, &e)
+	assert.Equal(expectedEntry.Key, e.Key)
+	assert.Equal(expectedEntry.Value, e.Value)
+	assert.Equal(expectedEntry.Checksum, e.Checksum)
+	assert.Equal(expectedEntry.Offset, e.Offset)
+	assert.Equal(*expectedEntry.Expiry, *e.Expiry)
+}

internal/data/codec/encoder.go

@@ -6,13 +6,15 @@ import (
 	"io"
 
 	"github.com/pkg/errors"
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 )
 
 const (
 	keySize      = 4
 	valueSize    = 8
 	checksumSize = 4
+	ttlSize      = 8
+	MetaInfoSize = keySize + valueSize + checksumSize + ttlSize
 )
 
 // NewEncoder creates a streaming Entry encoder.
@@ -49,9 +51,19 @@ func (e *Encoder) Encode(msg internal.Entry) (int64, error) {
 		return 0, errors.Wrap(err, "failed writing checksum data")
 	}
 
+	bufTTL := bufKeyValue[:ttlSize]
+	if msg.Expiry == nil {
+		binary.BigEndian.PutUint64(bufTTL, uint64(0))
+	} else {
+		binary.BigEndian.PutUint64(bufTTL, uint64(msg.Expiry.Unix()))
+	}
+	if _, err := e.w.Write(bufTTL); err != nil {
+		return 0, errors.Wrap(err, "failed writing ttl data")
+	}
+
 	if err := e.w.Flush(); err != nil {
 		return 0, errors.Wrap(err, "failed flushing data")
 	}
 
-	return int64(keySize + valueSize + len(msg.Key) + len(msg.Value) + checksumSize), nil
+	return int64(keySize + valueSize + len(msg.Key) + len(msg.Value) + checksumSize + ttlSize), nil
 }

internal/data/codec/encoder_test.go

@@ -4,8 +4,9 @@ import (
 	"bytes"
 	"encoding/hex"
 	"testing"
+	"time"
 
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -14,15 +15,17 @@ func TestEncode(t *testing.T) {
 	assert := assert.New(t)
 
 	var buf bytes.Buffer
+	mockTime := time.Date(2020, 10, 1, 0, 0, 0, 0, time.UTC)
 	encoder := NewEncoder(&buf)
 	_, err := encoder.Encode(internal.Entry{
 		Key:      []byte("mykey"),
 		Value:    []byte("myvalue"),
 		Checksum: 414141,
 		Offset:   424242,
+		Expiry:   &mockTime,
 	})
 
-	expectedHex := "0000000500000000000000076d796b65796d7976616c7565000651bd"
+	expectedHex := "0000000500000000000000076d796b65796d7976616c7565000651bd000000005f751c00"
 	if assert.NoError(err) {
 		assert.Equal(expectedHex, hex.EncodeToString(buf.Bytes()))
 	}

internal/data/datafile.go

@@ -7,8 +7,8 @@ import (
 	"sync"
 
 	"github.com/pkg/errors"
-	"github.com/prologic/bitcask/internal"
-	"github.com/prologic/bitcask/internal/data/codec"
+	"git.mills.io/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal/data/codec"
 	"golang.org/x/exp/mmap"
 )
 
@@ -19,8 +19,6 @@ const (
 var (
 	errReadonly  = errors.New("error: read only datafile")
 	errReadError = errors.New("error: read error")
-
-	mxMemPool sync.RWMutex
 )
 
 // Datafile is an interface  that represents a readable and writeable datafile
@@ -50,7 +48,7 @@ type datafile struct {
 }
 
 // NewDatafile opens an existing datafile
-func NewDatafile(path string, id int, readonly bool, maxKeySize uint32, maxValueSize uint64) (Datafile, error) {
+func NewDatafile(path string, id int, readonly bool, maxKeySize uint32, maxValueSize uint64, fileMode os.FileMode) (Datafile, error) {
 	var (
 		r   *os.File
 		ra  *mmap.ReaderAt
@@ -61,7 +59,7 @@ func NewDatafile(path string, id int, readonly bool, maxKeySize uint32, maxValue
 	fn := filepath.Join(path, fmt.Sprintf(defaultDatafileFilename, id))
 
 	if !readonly {
-		w, err = os.OpenFile(fn, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0640)
+		w, err = os.OpenFile(fn, os.O_WRONLY|os.O_APPEND|os.O_CREATE, fileMode)
 		if err != nil {
 			return nil, err
 		}

internal/data/recover.go

@@ -0,0 +1,95 @@
+package data
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"git.mills.io/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal/config"
+	"git.mills.io/prologic/bitcask/internal/data/codec"
+)
+
+// CheckAndRecover checks and recovers the last datafile.
+// If the datafile isn't corrupted, this is a noop. If it is,
+// the longest non-corrupted prefix will be kept and the rest
+// will be *deleted*. Also, the index file is also *deleted* which
+// will be automatically recreated on next startup.
+func CheckAndRecover(path string, cfg *config.Config) error {
+	dfs, err := internal.GetDatafiles(path)
+	if err != nil {
+		return fmt.Errorf("scanning datafiles: %s", err)
+	}
+	if len(dfs) == 0 {
+		return nil
+	}
+	f := dfs[len(dfs)-1]
+	recovered, err := recoverDatafile(f, cfg)
+	if err != nil {
+		return fmt.Errorf("error recovering data file: %s", err)
+	}
+	if recovered {
+		if err := os.Remove(filepath.Join(path, "index")); err != nil {
+			return fmt.Errorf("error deleting the index on recovery: %s", err)
+		}
+	}
+	return nil
+}
+
+func recoverDatafile(path string, cfg *config.Config) (recovered bool, err error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return false, fmt.Errorf("opening the datafile: %s", err)
+	}
+	defer func() {
+		closeErr := f.Close()
+		if err == nil {
+			err = closeErr
+		}
+	}()
+	dir, file := filepath.Split(path)
+	rPath := filepath.Join(dir, fmt.Sprintf("%s.recovered", file))
+	fr, err := os.OpenFile(rPath, os.O_CREATE|os.O_WRONLY, os.ModePerm)
+	if err != nil {
+		return false, fmt.Errorf("creating the recovered datafile: %w", err)
+	}
+	defer func() {
+		closeErr := fr.Close()
+		if err == nil {
+			err = closeErr
+		}
+	}()
+
+	dec := codec.NewDecoder(f, cfg.MaxKeySize, cfg.MaxValueSize)
+	enc := codec.NewEncoder(fr)
+	e := internal.Entry{}
+
+	corrupted := false
+	for !corrupted {
+		_, err = dec.Decode(&e)
+		if err == io.EOF {
+			break
+		}
+		if codec.IsCorruptedData(err) {
+			corrupted = true
+			continue
+		}
+		if err != nil {
+			return false, fmt.Errorf("unexpected error while reading datafile: %w", err)
+		}
+		if _, err := enc.Encode(e); err != nil {
+			return false, fmt.Errorf("writing to recovered datafile: %w", err)
+		}
+	}
+	if !corrupted {
+		if err := os.Remove(fr.Name()); err != nil {
+			return false, fmt.Errorf("can't remove temporal recovered datafile: %w", err)
+		}
+		return false, nil
+	}
+	if err := os.Rename(rPath, path); err != nil {
+		return false, fmt.Errorf("removing corrupted file: %s", err)
+	}
+	return true, nil
+}

internal/entry.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"hash/crc32"
+	"time"
 )
 
 // Entry represents a key/value in the database
@@ -10,15 +11,17 @@ type Entry struct {
 	Key      []byte
 	Offset   int64
 	Value    []byte
+	Expiry   *time.Time
 }
 
 // NewEntry creates a new `Entry` with the given `key` and `value`
-func NewEntry(key, value []byte) Entry {
+func NewEntry(key, value []byte, expiry *time.Time) Entry {
 	checksum := crc32.ChecksumIEEE(value)
 
 	return Entry{
 		Checksum: checksum,
 		Key:      key,
 		Value:    value,
+		Expiry:   expiry,
 	}
 }

internal/index/codec_index.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/pkg/errors"
 	art "github.com/plar/go-adaptive-radix-tree"
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 )
 
 var (
@@ -34,7 +34,7 @@ func readKeyBytes(r io.Reader, maxKeySize uint32) ([]byte, error) {
 		return nil, errors.Wrap(errTruncatedKeySize, err.Error())
 	}
 	size := binary.BigEndian.Uint32(s)
-	if size > uint32(maxKeySize) {
+	if maxKeySize > 0 && size > uint32(maxKeySize) {
 		return nil, errKeySizeTooLarge
 	}
 
@@ -117,11 +117,7 @@ func writeIndex(t art.Tree, w io.Writer) (err error) {
 
 		item := node.Value().(internal.Item)
 		err := writeItem(item, w)
-		if err != nil {
-			return false
-		}
-
-		return true
+		return err == nil
 	})
 	return
 }

internal/index/codec_index_test.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/pkg/errors"
 	art "github.com/plar/go-adaptive-radix-tree"
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 )
 
 const (

internal/index/index.go

@@ -4,7 +4,7 @@ import (
 	"os"
 
 	art "github.com/plar/go-adaptive-radix-tree"
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 )
 
 // Indexer is an interface for loading and saving the index (an Adaptive Radix Tree)

internal/index/ttl_index.go

@@ -0,0 +1,71 @@
+package index
+
+import (
+	"encoding/binary"
+	"io"
+	"os"
+	"time"
+
+	art "github.com/plar/go-adaptive-radix-tree"
+	"git.mills.io/prologic/bitcask/internal"
+)
+
+type ttlIndexer struct{}
+
+func NewTTLIndexer() Indexer {
+	return ttlIndexer{}
+}
+
+func (i ttlIndexer) Save(t art.Tree, path string) error {
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	buf := make([]byte, int64Size)
+	for it := t.Iterator(); it.HasNext(); {
+		node, err := it.Next()
+		if err != nil {
+			return err
+		}
+		// save key
+		err = writeBytes(node.Key(), f)
+		if err != nil {
+			return err
+		}
+		// save key ttl
+		binary.BigEndian.PutUint64(buf, uint64(node.Value().(time.Time).Unix()))
+		_, err = f.Write(buf)
+		if err != nil {
+			return err
+		}
+	}
+	return f.Sync()
+}
+
+func (i ttlIndexer) Load(path string, maxKeySize uint32) (art.Tree, bool, error) {
+	t := art.New()
+	if !internal.Exists(path) {
+		return t, false, nil
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return t, true, err
+	}
+	buf := make([]byte, int64Size)
+	for {
+		key, err := readKeyBytes(f, maxKeySize)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return t, true, err
+		}
+		_, err = io.ReadFull(f, buf)
+		if err != nil {
+			return t, true, err
+		}
+		expiry := time.Unix(int64(binary.BigEndian.Uint64(buf)), 0).UTC()
+		t.Insert(key, expiry)
+	}
+	return t, true, nil
+}

internal/index/ttl_index_test.go

@@ -0,0 +1,54 @@
+package index
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	art "github.com/plar/go-adaptive-radix-tree"
+	assert2 "github.com/stretchr/testify/assert"
+)
+
+func Test_TTLIndexer(t *testing.T) {
+	assert := assert2.New(t)
+	tempDir, err := ioutil.TempDir("", "bitcask")
+	assert.NoError(err)
+	defer os.RemoveAll(tempDir)
+
+	currTime := time.Date(2020, 12, 27, 0, 0, 0, 0, time.UTC)
+	trie := art.New()
+
+	t.Run("LoadEmpty", func(t *testing.T) {
+		newTrie, found, err := NewTTLIndexer().Load(filepath.Join(tempDir, "ttl_index"), 4)
+		assert.NoError(err)
+		assert.False(found)
+		assert.Equal(trie, newTrie)
+	})
+
+	t.Run("Save", func(t *testing.T) {
+		trie.Insert([]byte("key"), currTime)
+		err := NewTTLIndexer().Save(trie, filepath.Join(tempDir, "ttl_index"))
+		assert.NoError(err)
+		trie.Insert([]byte("foo"), currTime.Add(24*time.Hour))
+		err = NewTTLIndexer().Save(trie, filepath.Join(tempDir, "ttl_index"))
+		assert.NoError(err)
+		trie.Insert([]byte("key"), currTime.Add(-24*time.Hour))
+		err = NewTTLIndexer().Save(trie, filepath.Join(tempDir, "ttl_index"))
+		assert.NoError(err)
+	})
+
+	t.Run("Load", func(t *testing.T) {
+		newTrie, found, err := NewTTLIndexer().Load(filepath.Join(tempDir, "ttl_index"), 4)
+		assert.NoError(err)
+		assert.True(found)
+		assert.Equal(2, newTrie.Size())
+		value, found := newTrie.Search([]byte("key"))
+		assert.True(found)
+		assert.Equal(currTime.Add(-24*time.Hour), value)
+		value, found = newTrie.Search([]byte("foo"))
+		assert.True(found)
+		assert.Equal(currTime.Add(24*time.Hour), value)
+	})
+}

internal/metadata/metadata.go

@@ -0,0 +1,22 @@
+package metadata
+
+import (
+	"os"
+
+	"git.mills.io/prologic/bitcask/internal"
+)
+
+type MetaData struct {
+	IndexUpToDate    bool  `json:"index_up_to_date"`
+	ReclaimableSpace int64 `json:"reclaimable_space"`
+}
+
+func (m *MetaData) Save(path string, mode os.FileMode) error {
+	return internal.SaveJsonToFile(m, path, mode)
+}
+
+func Load(path string) (*MetaData, error) {
+	var m MetaData
+	err := internal.LoadFromJsonFile(path, &m)
+	return &m, err
+}

internal/mocks/datafile.go

@@ -2,7 +2,7 @@
 
 package mocks
 
-import internal "github.com/prologic/bitcask/internal"
+import internal "git.mills.io/prologic/bitcask/internal"
 import mock "github.com/stretchr/testify/mock"
 
 // Datafile is an autogenerated mock type for the Datafile type

internal/utils.go

@@ -1,7 +1,9 @@
 package internal
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
@@ -33,7 +35,8 @@ func DirSize(path string) (int64, error) {
 
 // GetDatafiles returns a list of all data files stored in the database path
 // given by `path`. All datafiles are identified by the the glob `*.data` and
-// the basename is represented by an monotomic increasing integer.
+// the basename is represented by a monotonic increasing integer.
+// The returned files are *sorted* in increasing order.
 func GetDatafiles(path string) ([]string, error) {
 	fns, err := filepath.Glob(fmt.Sprintf("%s/*.data", path))
 	if err != nil {
@@ -62,3 +65,48 @@ func ParseIds(fns []string) ([]int, error) {
 	sort.Ints(ids)
 	return ids, nil
 }
+
+// Copy copies source contents to destination
+func Copy(src, dst string, exclude []string) error {
+	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+		relPath := strings.Replace(path, src, "", 1)
+		if relPath == "" {
+			return nil
+		}
+		for _, e := range exclude {
+			matched, err := filepath.Match(e, info.Name())
+			if err != nil {
+				return err
+			}
+			if matched {
+				return nil
+			}
+		}
+		if info.IsDir() {
+			return os.Mkdir(filepath.Join(dst, relPath), info.Mode())
+		}
+		var data, err1 = ioutil.ReadFile(filepath.Join(src, relPath))
+		if err1 != nil {
+			return err1
+		}
+		return ioutil.WriteFile(filepath.Join(dst, relPath), data, info.Mode())
+	})
+}
+
+// SaveJsonToFile converts v into json and store in file identified by path
+func SaveJsonToFile(v interface{}, path string, mode os.FileMode) error {
+	b, err := json.Marshal(v)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(path, b, mode)
+}
+
+// LoadFromJsonFile reads file located at `path` and put its content in json format in v
+func LoadFromJsonFile(path string, v interface{}) error {
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(b, v)
+}

internal/utils_test.go

@@ -0,0 +1,108 @@
+package internal
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_Copy(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("CopyDir", func(t *testing.T) {
+		tempsrc, err := ioutil.TempDir("", "test")
+		assert.NoError(err)
+		defer os.RemoveAll(tempsrc)
+		var f *os.File
+
+		tempdir, err := ioutil.TempDir(tempsrc, "")
+		assert.NoError(err)
+
+		f, err = os.OpenFile(filepath.Join(tempsrc, "file1"), os.O_WRONLY|os.O_CREATE, 0755)
+		assert.NoError(err)
+		n, err := f.WriteString("test123")
+		assert.Equal(7, n)
+		assert.NoError(err)
+		f.Close()
+
+		f, err = os.OpenFile(filepath.Join(tempsrc, "file2"), os.O_WRONLY|os.O_CREATE, 0755)
+		assert.NoError(err)
+		n, err = f.WriteString("test1234")
+		assert.Equal(8, n)
+		assert.NoError(err)
+		f.Close()
+
+		f, err = os.OpenFile(filepath.Join(tempsrc, "file3"), os.O_WRONLY|os.O_CREATE, 0755)
+		assert.NoError(err)
+		f.Close()
+
+		tempdst, err := ioutil.TempDir("", "backup")
+		assert.NoError(err)
+		defer os.RemoveAll(tempdst)
+		err = Copy(tempsrc, tempdst, []string{"file3"})
+		assert.NoError(err)
+		buf := make([]byte, 10)
+
+		exists := Exists(filepath.Join(tempdst, filepath.Base(tempdir)))
+		assert.Equal(true, exists)
+
+		f, err = os.Open(filepath.Join(tempdst, "file1"))
+		assert.NoError(err)
+		n, err = f.Read(buf[:7])
+		assert.NoError(err)
+		assert.Equal(7, n)
+		assert.Equal([]byte("test123"), buf[:7])
+		_, err = f.Read(buf)
+		assert.Equal(io.EOF, err)
+		f.Close()
+
+		f, err = os.Open(filepath.Join(tempdst, "file2"))
+		assert.NoError(err)
+		n, err = f.Read(buf[:8])
+		assert.NoError(err)
+		assert.Equal(8, n)
+		assert.Equal([]byte("test1234"), buf[:8])
+		_, err = f.Read(buf)
+		assert.Equal(io.EOF, err)
+		f.Close()
+
+		exists = Exists(filepath.Join(tempdst, "file3"))
+		assert.Equal(false, exists)
+	})
+}
+
+func Test_SaveAndLoad(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("save and load", func(t *testing.T) {
+		tempdir, err := ioutil.TempDir("", "bitcask")
+		assert.NoError(err)
+		defer os.RemoveAll(tempdir)
+		type test struct {
+			Value bool `json:"value"`
+		}
+		m := test{Value: true}
+		err = SaveJsonToFile(&m, filepath.Join(tempdir, "meta.json"), 0755)
+		assert.NoError(err)
+		m1 := test{}
+		err = LoadFromJsonFile(filepath.Join(tempdir, "meta.json"), &m1)
+		assert.NoError(err)
+		assert.Equal(m, m1)
+	})
+
+	t.Run("save and load error", func(t *testing.T) {
+		tempdir, err := ioutil.TempDir("", "bitcask")
+		assert.NoError(err)
+		defer os.RemoveAll(tempdir)
+		type test struct {
+			Value bool `json:"value"`
+		}
+		err = SaveJsonToFile(make(chan int), filepath.Join(tempdir, "meta.json"), 0755)
+		assert.Error(err)
+		m1 := test{}
+		err = LoadFromJsonFile(filepath.Join(tempdir, "meta.json"), &m1)
+		assert.Error(err)
+	})
+}

options.go

@@ -1,8 +1,18 @@
 package bitcask
 
-import "github.com/prologic/bitcask/internal/config"
+import (
+	"os"
+
+	"git.mills.io/prologic/bitcask/internal/config"
+)
 
 const (
+	// DefaultDirFileModeBeforeUmask is the default os.FileMode used when creating directories
+	DefaultDirFileModeBeforeUmask = os.FileMode(0700)
+
+	// DefaultFileFileModeBeforeUmask is the default os.FileMode used when creating files
+	DefaultFileFileModeBeforeUmask = os.FileMode(0600)
+
 	// DefaultMaxDatafileSize is the default maximum datafile size in bytes
 	DefaultMaxDatafileSize = 1 << 20 // 1MB
 
@@ -14,11 +24,54 @@ const (
 
 	// DefaultSync is the default file synchronization action
 	DefaultSync = false
+
+	// DefaultAutoRecovery is the default auto-recovery action.
+
+	CurrentDBVersion = uint32(1)
 )
 
 // Option is a function that takes a config struct and modifies it
 type Option func(*config.Config) error
 
+func withConfig(src *config.Config) Option {
+	return func(cfg *config.Config) error {
+		cfg.MaxDatafileSize = src.MaxDatafileSize
+		cfg.MaxKeySize = src.MaxKeySize
+		cfg.MaxValueSize = src.MaxValueSize
+		cfg.Sync = src.Sync
+		cfg.AutoRecovery = src.AutoRecovery
+		cfg.DirFileModeBeforeUmask = src.DirFileModeBeforeUmask
+		cfg.FileFileModeBeforeUmask = src.FileFileModeBeforeUmask
+		return nil
+	}
+}
+
+// WithAutoRecovery sets auto recovery of data and index file recreation.
+// IMPORTANT: This flag MUST BE used only if a proper backup was made of all
+// the existing datafiles.
+func WithAutoRecovery(enabled bool) Option {
+	return func(cfg *config.Config) error {
+		cfg.AutoRecovery = enabled
+		return nil
+	}
+}
+
+// WithDirFileModeBeforeUmask sets the FileMode used for each new file created.
+func WithDirFileModeBeforeUmask(mode os.FileMode) Option {
+	return func(cfg *config.Config) error {
+		cfg.DirFileModeBeforeUmask = mode
+		return nil
+	}
+}
+
+// WithFileFileModeBeforeUmask sets the FileMode used for each new file created.
+func WithFileFileModeBeforeUmask(mode os.FileMode) Option {
+	return func(cfg *config.Config) error {
+		cfg.FileFileModeBeforeUmask = mode
+		return nil
+	}
+}
+
 // WithMaxDatafileSize sets the maximum datafile size option
 func WithMaxDatafileSize(size int) Option {
 	return func(cfg *config.Config) error {
@@ -54,9 +107,12 @@ func WithSync(sync bool) Option {
 
 func newDefaultConfig() *config.Config {
 	return &config.Config{
-		MaxDatafileSize: DefaultMaxDatafileSize,
-		MaxKeySize:      DefaultMaxKeySize,
-		MaxValueSize:    DefaultMaxValueSize,
-		Sync:            DefaultSync,
+		MaxDatafileSize:         DefaultMaxDatafileSize,
+		MaxKeySize:              DefaultMaxKeySize,
+		MaxValueSize:            DefaultMaxValueSize,
+		Sync:                    DefaultSync,
+		DirFileModeBeforeUmask:  DefaultDirFileModeBeforeUmask,
+		FileFileModeBeforeUmask: DefaultFileFileModeBeforeUmask,
+		DBVersion:               CurrentDBVersion,
 	}
 }

scripts/migrations/v0_to_v1.go

@@ -0,0 +1,159 @@
+package migrations
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+
+	"git.mills.io/prologic/bitcask/internal"
+)
+
+const (
+	keySize                 = 4
+	valueSize               = 8
+	checksumSize            = 4
+	ttlSize                 = 8
+	defaultDatafileFilename = "%09d.data"
+)
+
+func ApplyV0ToV1(dir string, maxDatafileSize int) error {
+	temp, err := prepare(dir)
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(temp)
+	err = apply(dir, temp, maxDatafileSize)
+	if err != nil {
+		return err
+	}
+	return cleanup(dir, temp)
+}
+
+func prepare(dir string) (string, error) {
+	return ioutil.TempDir(dir, "migration")
+}
+
+func apply(dir, temp string, maxDatafileSize int) error {
+	datafilesPath, err := internal.GetDatafiles(dir)
+	if err != nil {
+		return err
+	}
+	var id, newOffset int
+	datafile, err := getNewDatafile(temp, id)
+	if err != nil {
+		return err
+	}
+	id++
+	for _, p := range datafilesPath {
+		df, err := os.Open(p)
+		if err != nil {
+			return err
+		}
+		var off int64
+		for {
+			entry, err := getSingleEntry(df, off)
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				return err
+			}
+			if newOffset+len(entry) > maxDatafileSize {
+				err = datafile.Sync()
+				if err != nil {
+					return err
+				}
+				datafile, err = getNewDatafile(temp, id)
+				if err != nil {
+					return err
+				}
+				id++
+				newOffset = 0
+			}
+			newEntry := make([]byte, len(entry)+ttlSize)
+			copy(newEntry[:len(entry)], entry)
+			n, err := datafile.Write(newEntry)
+			if err != nil {
+				return err
+			}
+			newOffset += n
+			off += int64(len(entry))
+		}
+	}
+	return datafile.Sync()
+}
+
+func cleanup(dir, temp string) error {
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return err
+	}
+	for _, file := range files {
+		if !file.IsDir() {
+			err := os.RemoveAll(path.Join([]string{dir, file.Name()}...))
+			if err != nil {
+				return err
+			}
+		}
+	}
+	files, err = ioutil.ReadDir(temp)
+	if err != nil {
+		return err
+	}
+	for _, file := range files {
+		err := os.Rename(
+			path.Join([]string{temp, file.Name()}...),
+			path.Join([]string{dir, file.Name()}...),
+		)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func getNewDatafile(path string, id int) (*os.File, error) {
+	fn := filepath.Join(path, fmt.Sprintf(defaultDatafileFilename, id))
+	return os.OpenFile(fn, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0640)
+}
+
+func getSingleEntry(f *os.File, offset int64) ([]byte, error) {
+	prefixBuf, err := readPrefix(f, offset)
+	if err != nil {
+		return nil, err
+	}
+	actualKeySize, actualValueSize := getKeyValueSize(prefixBuf)
+	entryBuf, err := read(f, uint64(actualKeySize)+actualValueSize+checksumSize, offset+keySize+valueSize)
+	if err != nil {
+		return nil, err
+	}
+	return append(prefixBuf, entryBuf...), nil
+}
+
+func readPrefix(f *os.File, offset int64) ([]byte, error) {
+	prefixBuf := make([]byte, keySize+valueSize)
+	_, err := f.ReadAt(prefixBuf, offset)
+	if err != nil {
+		return nil, err
+	}
+	return prefixBuf, nil
+}
+
+func read(f *os.File, bufSize uint64, offset int64) ([]byte, error) {
+	buf := make([]byte, bufSize)
+	_, err := f.ReadAt(buf, offset)
+	if err != nil {
+		return nil, err
+	}
+	return buf, nil
+}
+
+func getKeyValueSize(buf []byte) (uint32, uint64) {
+	actualKeySize := binary.BigEndian.Uint32(buf[:keySize])
+	actualValueSize := binary.BigEndian.Uint64(buf[keySize:])
+	return actualKeySize, actualValueSize
+}

scripts/migrations/v0_to_v1_test.go

@@ -0,0 +1,58 @@
+package migrations
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_ApplyV0ToV1(t *testing.T) {
+	assert := assert.New(t)
+	testdir, err := ioutil.TempDir("/tmp", "bitcask")
+	assert.NoError(err)
+	defer os.RemoveAll(testdir)
+	w0, err := os.OpenFile(filepath.Join(testdir, "000000000.data"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
+	assert.NoError(err)
+	w1, err := os.OpenFile(filepath.Join(testdir, "000000001.data"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
+	assert.NoError(err)
+	w2, err := os.OpenFile(filepath.Join(testdir, "000000002.data"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
+	assert.NoError(err)
+	defer w0.Close()
+	defer w1.Close()
+	defer w2.Close()
+	buf := make([]byte, 104)
+	binary.BigEndian.PutUint32(buf[:4], 5)
+	binary.BigEndian.PutUint64(buf[4:12], 7)
+	copy(buf[12:28], "mykeymyvalue0AAA")
+	binary.BigEndian.PutUint32(buf[28:32], 3)
+	binary.BigEndian.PutUint64(buf[32:40], 5)
+	copy(buf[40:52], "keyvalue0BBB")
+	_, err = w0.Write(buf[:52])
+	assert.NoError(err)
+	_, err = w1.Write(buf[:52])
+	assert.NoError(err)
+	_, err = w2.Write(buf[:52])
+	assert.NoError(err)
+	err = ApplyV0ToV1(testdir, 104)
+	assert.NoError(err)
+	r0, err := os.Open(filepath.Join(testdir, "000000000.data"))
+	assert.NoError(err)
+	defer r0.Close()
+	n, err := io.ReadFull(r0, buf)
+	assert.NoError(err)
+	assert.Equal(104, n)
+	assert.Equal("0000000500000000000000076d796b65796d7976616c75653041414100000000000000000000000300000000000000056b657976616c75653042424200000000000000000000000500000000000000076d796b65796d7976616c7565304141410000000000000000", hex.EncodeToString(buf))
+	r1, err := os.Open(filepath.Join(testdir, "000000001.data"))
+	assert.NoError(err)
+	defer r1.Close()
+	n, err = io.ReadFull(r1, buf[:100])
+	assert.NoError(err)
+	assert.Equal(100, n)
+	assert.Equal("0000000300000000000000056b657976616c75653042424200000000000000000000000500000000000000076d796b65796d7976616c75653041414100000000000000000000000300000000000000056b657976616c7565304242420000000000000000", hex.EncodeToString(buf[:100]))
+}

tools/release.sh

@@ -1,16 +1,19 @@
-#!/bin/sh
+#!/bin/bash
 
 # Get the highest tag number
 VERSION="$(git describe --abbrev=0 --tags)"
 VERSION=${VERSION:-'0.0.0'}
 
 # Get number parts
-MAJOR="${VERSION%%.*}"; VERSION="${VERSION#*.}"
-MINOR="${VERSION%%.*}"; VERSION="${VERSION#*.}"
-PATCH="${VERSION%%.*}"; VERSION="${VERSION#*.}"
+MAJOR="${VERSION%%.*}"
+VERSION="${VERSION#*.}"
+MINOR="${VERSION%%.*}"
+VERSION="${VERSION#*.}"
+PATCH="${VERSION%%.*}"
+VERSION="${VERSION#*.}"
 
 # Increase version
-PATCH=$((PATCH+1))
+PATCH=$((PATCH + 1))
 
 TAG="${1}"
 
@@ -20,6 +23,10 @@ fi
 
 echo "Releasing ${TAG} ..."
 
+git-chglog --next-tag="${TAG}" --output CHANGELOG.md
+git commit -a -m "Update CHANGELOG for ${TAG}"
 git tag -a -s -m "Release ${TAG}" "${TAG}"
 git push --tags
-goreleaser release --rm-dist
+goreleaser release \
+  --rm-dist \
+  --release-notes <(git-chglog "${TAG}" | tail -n+5)