update sift to match docstring

cc @taigrr

This PR will _hopefully_ help to fix some critical isseus in the real world with several or more [Yarn.social](https://yarn.social) pods running [yarnd](https://git.mills.io/yarnsocial/yarn) where starting back up after a power failure or crash can sometimes result in an empty `config.json` or empty `meta.json` or both!

I'm not actually sure how this can arise, and as yet I haven't been able to reproduce it (_I can only assume this has to be failures cases outside of our control_); but in any case the application and database is recoverable by simply `rm config.json` and/or `rm meta.json`.

So this PR makes errors loading the config and metadata first-class errors and exported error types that consumers of the library can use to perform automated recovery without requiring human intervention.

Basiclaly in this case it's no big deal we lost the database config of metadata, we can simply carry on.

Co-authored-by: James Mills <prologic@shortcircuit.net.au>
Reviewed-on: https://git.mills.io/prologic/bitcask/pulls/241
Co-authored-by: James Mills <james@mills.io>
Co-committed-by: James Mills <james@mills.io>

.chglog/config.yml

@@ -3,7 +3,7 @@ style: Github
 template: CHANGELOG.tpl.md
 info:
   title: CHANGELOG
-  repository_url: https://github.com/prologic/bitcask
+  repository_url: https://git.mills.io/prologic/bitcask
 options:
   commits:
     filters:

.dependabot/config.yml

@@ -1,8 +0,0 @@
-version: 1
-update_configs:
-  - package_manager: "go:modules"
-    directory: "/"
-    update_schedule: "daily"
-  - package_manager: "docker"
-    directory: "/"
-    update_schedule: "weekly"

.drone.yml

@@ -0,0 +1,28 @@
+---
+kind: pipeline
+name: default
+
+steps:
+  - name: build & run tests
+    image: r.mills.io/prologic/golang-alpine
+    commands:
+      - make build
+      - make test
+
+  - name: notify
+    image: plugins/webhook
+    settings:
+      urls:
+        - https://msgbus.mills.io/ci.mills.io
+    when:
+      status:
+        - success
+        - failure
+
+trigger:
+  branch:
+    - master
+  event:
+    - tag
+    - push
+    - pull_request

.github/FUNDING.yml

@@ -1,2 +0,0 @@
-github: prologic
-patreon: prologic

.github/labeler.yml

@@ -1,9 +0,0 @@
-documentation:
-  - "**/*.md"
-
-tests:
-  - "**/*_test.go"
-
-dependencies:
-  - go.mod
-  - go.sum

.github/workflows/auto-approve.yml

@@ -1,11 +0,0 @@
-name: Auto approve
-on: [pull_request]
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: hmarr/auto-approve-action@v2.0.0
-      if: github.actor == 'dependabot[bot]' || github.actor == 'dependabot-preview[bot]'
-      with:
-        github-token: "${{ secrets.GITHUB_TOKEN }}"
-

.github/workflows/autoassign.yml

@@ -1,10 +0,0 @@
-name: AutoAssigner
-on: [pull_request]
-jobs:
-  assignAuthor:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: samspills/assign-pr-to-author@v1.0
-        if: github.event_name == 'pull_request' && github.event.action == 'opened'
-        with:
-          repo-token: '${{ secrets.GITHUB_TOKEN }}'

.github/workflows/codecov.yml

@@ -1,23 +0,0 @@
-name: Coverage
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-jobs:
-  test:
-    name: Test and Report
-    runs-on: ubuntu-latest
-    steps:
-      - name: Setup Go
-        uses: actions/setup-go@v1
-        with:
-          go-version: 1.13.x
-      - name: Checkout
-        uses: actions/checkout@v1
-      - name: Test
-        run: go test -v -cover -coverprofile=coverage.txt -covermode=atomic -race .
-      - name: Report
-        uses: codecov/codecov-action@v1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/docker.yml

@@ -1,15 +0,0 @@
-name: Docker
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-jobs:
-  build:
-    name: Build Image
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v1
-      - name: Build
-        run: docker build -t bitcask .

.github/workflows/go.yml

@@ -1,26 +0,0 @@
-name: Go
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-jobs:
-  test:
-    name: Build and Test
-    strategy:
-      matrix:
-        go-version: [1.12.x, 1.13.x, 1.14.x]
-        platform: [ubuntu-latest, macos-latest, windows-latest]
-    runs-on: ${{ matrix.platform }}
-    steps:
-      - name: Setup Go ${{ matrix.go-version }}
-        uses: actions/setup-go@v1
-        with:
-          go-version: ${{ matrix.go-version }}
-        id: go
-      - name: Checkout
-        uses: actions/checkout@v1
-      - name: Build
-        run: go build -v .
-      - name: Test
-        run: go test -v -race .

.github/workflows/greetings.yml

@@ -1,42 +0,0 @@
-name: Greetings
-on: [pull_request, issues]
-jobs:
-  greeting:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/first-interaction@v1
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        issue-message: |-
-          Hello!
-          
-          Welcome to the Bitcask project! Someone will respond to your issue pretty quickly,
-          the author is pretty responsive 😀 In the meantime; please make sure you have read
-          the [Contributing](https://github.com/prologic/bitcask/blob/master/CONTRIBUTING.md)
-          and [Code of Conduct](https://github.com/prologic/bitcask/blob/master/CODE_OF_CONDUCT.md)
-          documents.
-          
-          If possible please also make sure your Bug Report or Feature Request is clearly defined
-          with either examples or a reproducible case (_if a bug_).
-          
-          Thank you 😃
-        pr-message: |-
-          Hello!
-          
-          Welcome to the Bitcask project!
-          
-          Thank you for your Pull Request and Contribution! We highly value all contributions to this Project!
-          Your Pull Request will be reviewed shortly! The author is pretty responsive 😀
-           In the meantime; please make sure you have read
-          the [Contributing](https://github.com/prologic/bitcask/blob/master/CONTRIBUTING.md)
-          and [Code of Conduct](https://github.com/prologic/bitcask/blob/master/CODE_OF_CONDUCT.md)
-          documents.
-          
-          Please also ensure that your PR passes all the CI/CD tests -- They will appear in your PR
-          towards the bottom just above the comment box.
-          
-          Also in addition, if you haven't already; please ammend your PR by modifying the
-          [AUTHORS](https://github.com/prologic/bitcask/blob/master/AUTHORS) file and adding
-          yourself to it! We like to recognize and peserve in Git history all contributors!
-          
-          Thank you 😃

.github/workflows/label.yml

@@ -1,9 +0,0 @@
-name: Labeler
-on: [pull_request]
-jobs:
-  label:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/labeler@v2
-      with:
-        repo-token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/reviewdog.yml

@@ -1,32 +0,0 @@
-name: ReviewDog
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-jobs:
-  golangci-lint:
-    name: runner / golangci-lint
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: reviewdog/action-golangci-lint@v1
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-  misspell:
-      name: runner / misspell
-      runs-on: ubuntu-latest
-      steps:
-        - uses: actions/checkout@v1
-        - uses: reviewdog/action-misspell@v1
-          with:
-            github_token: ${{ secrets.GItHUB_TOKEN }}
-  shellcheck:
-      name: runner / shellcheck
-      runs-on: ubuntu-latest
-      steps:
-        - uses: actions/checkout@v1
-        - uses: reviewdog/action-shellcheck@v1
-          with:
-            github_token: ${{ secrets.GITHUB_TOKEN }}
-            reporter: github-pr-review

.github/workflows/stale.yml

@@ -1,29 +0,0 @@
-name: Mark stale issues and pull requests
-
-on:
-  schedule:
-  - cron: "0 0 * * *"
-
-jobs:
-  stale:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/stale@v1
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-issue-message: |-
-          This Issue has gone stale and will be closed automatically.
-          
-          If this is incorrect, please reopen and add the label `on hold`.
-          
-          Thank you.
-        stale-pr-message: |-
-          This Pull Request has gone stale and will be closed automatically.
-          
-          If this is incorrect, please reopen and add the label `on hold`.
-          
-          Thank you.
-        stale-issue-label: 'stale'
-        exempt-issue-label: 'on hold'
-        stale-pr-label: 'stale'
-        exempt-pr-label: 'on hold'

.gitignore

@@ -1,12 +1,15 @@
 *~*
-*.bak
 *.db
+*.bak
+**.envrc
 
 /tmp
 /dist
+/cacheDb
 /coverage.txt
 
 /bitcask
 /bitcaskd
+/bitcask_bench*
 /cmd/bitcask/bitcask
 /cmd/bitcaskd/bitcaskd

.goreleaser.yml

@@ -5,21 +5,35 @@ builds:
     binary: bitcask
     main: ./cmd/bitcask
     flags: -tags "static_build"
-    ldflags: -w -X github.com/prologic/bitcask/internal.Version={{.Version}} -X github.com/prologic/bitcask/internal.Commit={{.Commit}}
+    ldflags: -w -X git.mills.io/prologic/bitcask/internal.Version={{.Version}} -X git.mills.io/prologic/bitcask/internal.Commit={{.Commit}}
     env:
       - CGO_ENABLED=0
+    goos:
+      - darwin
+      - linux
+    goarch:
+      - amd64
+      - arm64
   -
     id: bitcaskd
     binary: bitcaskd
     main: ./cmd/bitcaskd
     flags: -tags "static_build"
-    ldflags: -w -X github.com/prologic/bitcask/internal.Version={{.Version}} -X github.com/prologic/bitcask/internal.Commit={{.Commit}}
+    ldflags: -w -X git.mills.io/prologic/bitcask/internal.Version={{.Version}} -X git.mills.io/prologic/bitcask/internal.Commit={{.Commit}}
     env:
       - CGO_ENABLED=0
+    goos:
+      - darwin
+      - linux
+    goarch:
+      - amd64
+      - arm64
 signs:
   - artifacts: checksum
 release:
-  github:
+  gitea:
     owner: prologic
     name: bitcask
   draft: true
+gitea_urls:
+  api: https://git.mills.io/api/v1/

AUTHORS

@@ -13,6 +13,9 @@ Kebert Xela kebertxela
 panyun panyun 
 shiniao <zhuzhezhe95@gmail.com>
 Whemoon Jang <palindrom615@gmail.com>
+Yash Chandra <yashschandra@gmail.com>
 Yury Fedorov orlangure
 o2gy84 <o2gy84@gmail.com>
 garsue <labs.garsue@gmail.com>
+biozz <ielfimov@gmail.com>
+jason3gb <jason3gb@gmail.com>

CHANGELOG.md

@@ -1,14 +1,145 @@
 
+<a name="v1.0.2"></a>
+## [v1.0.2](https://git.mills.io/prologic/bitcask/compare/v1.0.1...v1.0.2) (2021-11-01)
+
+### Bug Fixes
+
+* Fix a data race in Datafile.ReadAt()
+* Fix release tool
+
+
+<a name="v1.0.1"></a>
+## [v1.0.1](https://git.mills.io/prologic/bitcask/compare/v1.0.0...v1.0.1) (2021-10-31)
+
+### Features
+
+* Add ErrBadConfig and ErrBadMetadata as errors that consumers can check and use (#241)
+* Add key prefix matching to KEYS command (#237)
+
+### Updates
+
+* Update CHANGELOG for v1.0.1
+* Update image target
+
+
+<a name="v1.0.0"></a>
+## [v1.0.0](https://git.mills.io/prologic/bitcask/compare/1.0.0...v1.0.0) (2021-07-24)
+
+### Updates
+
+* Update CHANGELOG for v1.0.0
+
+
+<a name="1.0.0"></a>
+## [1.0.0](https://git.mills.io/prologic/bitcask/compare/v0.3.14...1.0.0) (2021-07-24)
+
+### Updates
+
+* Update CHANGELOG for 1.0.0
+* Update README
+
+
+<a name="v0.3.14"></a>
+## [v0.3.14](https://git.mills.io/prologic/bitcask/compare/v0.3.13...v0.3.14) (2021-07-21)
+
+### Bug Fixes
+
+* Fix runGC behaviour to correctly delete all expired keys (#229)
+* Fix missing push event
+* Fix how CI is triggered
+* Fix README Go Reference badge
+* Fix README badges
+
+### Features
+
+* Add RangeScan() support (#160)
+
+### Updates
+
+* Update CHANGELOG for v0.3.14
+
+
+<a name="v0.3.13"></a>
+## [v0.3.13](https://git.mills.io/prologic/bitcask/compare/v0.3.12...v0.3.13) (2021-07-16)
+
+### Bug Fixes
+
+* Fix paths used for temporary recovery iles to avoid crossing devices (#223)
+
+### Features
+
+* Add Drone CI config
+
+### Updates
+
+* Update CHANGELOG for v0.3.13
+
+
+<a name="v0.3.12"></a>
+## [v0.3.12](https://git.mills.io/prologic/bitcask/compare/v0.3.11...v0.3.12) (2021-07-13)
+
+### Updates
+
+* Update CHANGELOG for v0.3.12
+
+
+<a name="v0.3.11"></a>
+## [v0.3.11](https://git.mills.io/prologic/bitcask/compare/v0.3.10...v0.3.11) (2021-07-10)
+
+### Bug Fixes
+
+* Fix missing go.sum entries
+* Fix GoReleaser config
+* Fix go.sum
+
+### Documentation
+
+* Document good and possibly poor use-cases of Bitcask (#199)
+
+### Features
+
+* Add support for keys with ttl (#177)
+
+### Updates
+
+* Update CHANGELOG for v0.3.11
+
+
+<a name="v0.3.10"></a>
+## [v0.3.10](https://git.mills.io/prologic/bitcask/compare/v0.3.9...v0.3.10) (2020-12-18)
+
+### Bug Fixes
+
+* Fix a bug when MaxValueSize == 0 on Merge operations
+* Fix link to bitcask-bench
+* Fix CI (again)
+* Fix CI
+
+### Features
+
+* Add support for unlimited key/value sizes
+* Add a few more test cases for concurrent operations
+
+### Updates
+
+* Update CHANGELOG for v0.3.10
+* Update README.md
+
+
 <a name="v0.3.9"></a>
-## [v0.3.9](https://github.com/prologic/bitcask/compare/v0.3.8...v0.3.9) (2020-11-17)
+## [v0.3.9](https://git.mills.io/prologic/bitcask/compare/v0.3.8...v0.3.9) (2020-11-17)
 
 ### Bug Fixes
 
 * Fix a race condition around .Close() and .Sync()
 
+### Updates
+
+* Update CHANGELOG for v0.3.9
+
 
 <a name="v0.3.8"></a>
-## [v0.3.8](https://github.com/prologic/bitcask/compare/v0.3.7...v0.3.8) (2020-11-17)
+## [v0.3.8](https://git.mills.io/prologic/bitcask/compare/v0.3.7...v0.3.8) (2020-11-17)
 
 ### Updates
 
@@ -16,7 +147,7 @@
 
 
 <a name="v0.3.7"></a>
-## [v0.3.7](https://github.com/prologic/bitcask/compare/v0.3.6...v0.3.7) (2020-11-17)
+## [v0.3.7](https://git.mills.io/prologic/bitcask/compare/v0.3.6...v0.3.7) (2020-11-17)
 
 ### Updates
 
@@ -24,24 +155,24 @@
 
 
 <a name="v0.3.6"></a>
-## [v0.3.6](https://github.com/prologic/bitcask/compare/v0.3.5...v0.3.6) (2020-11-17)
+## [v0.3.6](https://git.mills.io/prologic/bitcask/compare/v0.3.5...v0.3.6) (2020-11-17)
 
 ### Bug Fixes
 
-* Fix typo in labeler ([#172](https://github.com/prologic/bitcask/issues/172))
+* Fix typo in labeler (#172)
 * Fix builds configuration for goreleaser
 * Fix (again) goreleaser config
 * Fix goreleaser config and improve release notes / changelog
-* Fix recoverDatafile error covering ([#162](https://github.com/prologic/bitcask/issues/162))
-* Fix loadIndex to be deterministic ([#115](https://github.com/prologic/bitcask/issues/115))
+* Fix recoverDatafile error covering (#162)
+* Fix loadIndex to be deterministic (#115)
 
 ### Features
 
-* Add configuration options for FileMode ([#183](https://github.com/prologic/bitcask/issues/183))
-* Add imports and log in example code ([#182](https://github.com/prologic/bitcask/issues/182))
+* Add configuration options for FileMode (#183)
+* Add imports and log in example code (#182)
 * Add empty changelog
 * Add DependaBot config
-* Add DeleteAll function ([#116](https://github.com/prologic/bitcask/issues/116))
+* Add DeleteAll function (#116)
 
 ### Updates
 
@@ -49,12 +180,12 @@
 * Update README.md
 * Update CHANGELOG for v0.3.6
 * Update CHANGELOG for v0.3.6
-* Update deps ([#140](https://github.com/prologic/bitcask/issues/140))
+* Update deps (#140)
 * Update README.md
 
 
 <a name="v0.3.5"></a>
-## [v0.3.5](https://github.com/prologic/bitcask/compare/v0.3.4...v0.3.5) (2019-10-20)
+## [v0.3.5](https://git.mills.io/prologic/bitcask/compare/v0.3.4...v0.3.5) (2019-10-20)
 
 ### Bug Fixes
 
@@ -65,27 +196,27 @@
 ### Features
 
 * Add *.db to ignore future accidental commits of a bitcask db to the repo
-* Add unit test for opening bad database with corrupted/invalid datafiles ([#105](https://github.com/prologic/bitcask/issues/105))
+* Add unit test for opening bad database with corrupted/invalid datafiles (#105)
 
 ### Updates
 
 * Update Drone CI test pipeline
 * Update README.md
-* Update to Go 1.13 and update README with new benchmarks ([#89](https://github.com/prologic/bitcask/issues/89))
+* Update to Go 1.13 and update README with new benchmarks (#89)
 * Update README.md
 
 
 <a name="v0.3.4"></a>
-## [v0.3.4](https://github.com/prologic/bitcask/compare/v0.3.3...v0.3.4) (2019-09-02)
+## [v0.3.4](https://git.mills.io/prologic/bitcask/compare/v0.3.3...v0.3.4) (2019-09-02)
 
 
 <a name="v0.3.3"></a>
-## [v0.3.3](https://github.com/prologic/bitcask/compare/v0.3.2...v0.3.3) (2019-09-02)
+## [v0.3.3](https://git.mills.io/prologic/bitcask/compare/v0.3.2...v0.3.3) (2019-09-02)
 
 ### Bug Fixes
 
-* Fix a bug wit the decoder passing the wrong value for the value's offset into the buffer ([#77](https://github.com/prologic/bitcask/issues/77))
-* Fix typo ([#65](https://github.com/prologic/bitcask/issues/65))
+* Fix a bug wit the decoder passing the wrong value for the value's offset into the buffer (#77)
+* Fix typo (#65)
 * Fix and cleanup some unnecessary internal sub-packages and duplication
 
 ### Updates
@@ -97,7 +228,7 @@
 
 
 <a name="v0.3.2"></a>
-## [v0.3.2](https://github.com/prologic/bitcask/compare/v0.3.1...v0.3.2) (2019-08-08)
+## [v0.3.2](https://git.mills.io/prologic/bitcask/compare/v0.3.1...v0.3.2) (2019-08-08)
 
 ### Updates
 
@@ -107,7 +238,7 @@
 
 
 <a name="v0.3.1"></a>
-## [v0.3.1](https://github.com/prologic/bitcask/compare/v0.3.0...v0.3.1) (2019-08-05)
+## [v0.3.1](https://git.mills.io/prologic/bitcask/compare/v0.3.0...v0.3.1) (2019-08-05)
 
 ### Updates
 
@@ -117,7 +248,7 @@
 
 
 <a name="v0.3.0"></a>
-## [v0.3.0](https://github.com/prologic/bitcask/compare/v0.2.2...v0.3.0) (2019-07-29)
+## [v0.3.0](https://git.mills.io/prologic/bitcask/compare/v0.2.2...v0.3.0) (2019-07-29)
 
 ### Updates
 
@@ -126,29 +257,29 @@
 
 
 <a name="v0.2.2"></a>
-## [v0.2.2](https://github.com/prologic/bitcask/compare/v0.2.1...v0.2.2) (2019-07-27)
+## [v0.2.2](https://git.mills.io/prologic/bitcask/compare/v0.2.1...v0.2.2) (2019-07-27)
 
 
 <a name="v0.2.1"></a>
-## [v0.2.1](https://github.com/prologic/bitcask/compare/v0.2.0...v0.2.1) (2019-07-25)
+## [v0.2.1](https://git.mills.io/prologic/bitcask/compare/v0.2.0...v0.2.1) (2019-07-25)
 
 
 <a name="v0.2.0"></a>
-## [v0.2.0](https://github.com/prologic/bitcask/compare/v0.1.7...v0.2.0) (2019-07-25)
+## [v0.2.0](https://git.mills.io/prologic/bitcask/compare/v0.1.7...v0.2.0) (2019-07-25)
 
 ### Bug Fixes
 
-* Fix issue(db file Merge issue in windows env): ([#15](https://github.com/prologic/bitcask/issues/15))
+* Fix issue(db file Merge issue in windows env): (#15)
 
 
 <a name="v0.1.7"></a>
-## [v0.1.7](https://github.com/prologic/bitcask/compare/v0.1.6...v0.1.7) (2019-07-19)
+## [v0.1.7](https://git.mills.io/prologic/bitcask/compare/v0.1.6...v0.1.7) (2019-07-19)
 
 ### Bug Fixes
 
-* Fix mismatched key casing. ([#12](https://github.com/prologic/bitcask/issues/12))
-* Fix outdated README ([#11](https://github.com/prologic/bitcask/issues/11))
-* Fix typos in bitcask.go docs ([#10](https://github.com/prologic/bitcask/issues/10))
+* Fix mismatched key casing. (#12)
+* Fix outdated README (#11)
+* Fix typos in bitcask.go docs (#10)
 
 ### Updates
 
@@ -157,16 +288,16 @@
 
 
 <a name="v0.1.6"></a>
-## [v0.1.6](https://github.com/prologic/bitcask/compare/v0.1.5...v0.1.6) (2019-04-01)
+## [v0.1.6](https://git.mills.io/prologic/bitcask/compare/v0.1.5...v0.1.6) (2019-04-01)
 
 ### Features
 
-* Add Development section to README documenting use of Protobuf and tooling required. [#6](https://github.com/prologic/bitcask/issues/6)
+* Add Development section to README documenting use of Protobuf and tooling required. #6
 * Add other badges from img.shields.io
 
 
 <a name="v0.1.5"></a>
-## [v0.1.5](https://github.com/prologic/bitcask/compare/v0.1.4...v0.1.5) (2019-03-30)
+## [v0.1.5](https://git.mills.io/prologic/bitcask/compare/v0.1.4...v0.1.5) (2019-03-30)
 
 ### Documentation
 
@@ -182,27 +313,27 @@
 
 
 <a name="v0.1.4"></a>
-## [v0.1.4](https://github.com/prologic/bitcask/compare/v0.1.3...v0.1.4) (2019-03-23)
+## [v0.1.4](https://git.mills.io/prologic/bitcask/compare/v0.1.3...v0.1.4) (2019-03-23)
 
 
 <a name="v0.1.3"></a>
-## [v0.1.3](https://github.com/prologic/bitcask/compare/v0.1.2...v0.1.3) (2019-03-23)
+## [v0.1.3](https://git.mills.io/prologic/bitcask/compare/v0.1.2...v0.1.3) (2019-03-23)
 
 
 <a name="v0.1.2"></a>
-## [v0.1.2](https://github.com/prologic/bitcask/compare/v0.1.1...v0.1.2) (2019-03-22)
+## [v0.1.2](https://git.mills.io/prologic/bitcask/compare/v0.1.1...v0.1.2) (2019-03-22)
 
 
 <a name="v0.1.1"></a>
-## [v0.1.1](https://github.com/prologic/bitcask/compare/v0.1.0...v0.1.1) (2019-03-22)
+## [v0.1.1](https://git.mills.io/prologic/bitcask/compare/v0.1.0...v0.1.1) (2019-03-22)
 
 
 <a name="v0.1.0"></a>
-## [v0.1.0](https://github.com/prologic/bitcask/compare/0.0.26...v0.1.0) (2019-03-22)
+## [v0.1.0](https://git.mills.io/prologic/bitcask/compare/0.0.26...v0.1.0) (2019-03-22)
 
 
 <a name="0.0.26"></a>
-## [0.0.26](https://github.com/prologic/bitcask/compare/0.0.25...0.0.26) (2019-03-21)
+## [0.0.26](https://git.mills.io/prologic/bitcask/compare/0.0.25...0.0.26) (2019-03-21)
 
 ### Features
 
@@ -215,7 +346,7 @@
 
 
 <a name="0.0.25"></a>
-## [0.0.25](https://github.com/prologic/bitcask/compare/0.0.24...0.0.25) (2019-03-21)
+## [0.0.25](https://git.mills.io/prologic/bitcask/compare/0.0.24...0.0.25) (2019-03-21)
 
 ### Features
 
@@ -229,11 +360,11 @@
 
 
 <a name="0.0.24"></a>
-## [0.0.24](https://github.com/prologic/bitcask/compare/0.0.23...0.0.24) (2019-03-20)
+## [0.0.24](https://git.mills.io/prologic/bitcask/compare/0.0.23...0.0.24) (2019-03-20)
 
 
 <a name="0.0.23"></a>
-## [0.0.23](https://github.com/prologic/bitcask/compare/0.0.22...0.0.23) (2019-03-20)
+## [0.0.23](https://git.mills.io/prologic/bitcask/compare/0.0.22...0.0.23) (2019-03-20)
 
 ### Features
 
@@ -241,27 +372,27 @@
 
 
 <a name="0.0.22"></a>
-## [0.0.22](https://github.com/prologic/bitcask/compare/0.0.21...0.0.22) (2019-03-18)
+## [0.0.22](https://git.mills.io/prologic/bitcask/compare/0.0.21...0.0.22) (2019-03-18)
 
 
 <a name="0.0.21"></a>
-## [0.0.21](https://github.com/prologic/bitcask/compare/0.0.20...0.0.21) (2019-03-18)
+## [0.0.21](https://git.mills.io/prologic/bitcask/compare/0.0.20...0.0.21) (2019-03-18)
 
 
 <a name="0.0.20"></a>
-## [0.0.20](https://github.com/prologic/bitcask/compare/0.0.19...0.0.20) (2019-03-17)
+## [0.0.20](https://git.mills.io/prologic/bitcask/compare/0.0.19...0.0.20) (2019-03-17)
 
 
 <a name="0.0.19"></a>
-## [0.0.19](https://github.com/prologic/bitcask/compare/0.0.18...0.0.19) (2019-03-17)
+## [0.0.19](https://git.mills.io/prologic/bitcask/compare/0.0.18...0.0.19) (2019-03-17)
 
 
 <a name="0.0.18"></a>
-## [0.0.18](https://github.com/prologic/bitcask/compare/0.0.17...0.0.18) (2019-03-16)
+## [0.0.18](https://git.mills.io/prologic/bitcask/compare/0.0.17...0.0.18) (2019-03-16)
 
 
 <a name="0.0.17"></a>
-## [0.0.17](https://github.com/prologic/bitcask/compare/0.0.16...0.0.17) (2019-03-16)
+## [0.0.17](https://git.mills.io/prologic/bitcask/compare/0.0.16...0.0.17) (2019-03-16)
 
 ### Features
 
@@ -269,11 +400,11 @@
 
 
 <a name="0.0.16"></a>
-## [0.0.16](https://github.com/prologic/bitcask/compare/0.0.15...0.0.16) (2019-03-16)
+## [0.0.16](https://git.mills.io/prologic/bitcask/compare/0.0.15...0.0.16) (2019-03-16)
 
 
 <a name="0.0.15"></a>
-## [0.0.15](https://github.com/prologic/bitcask/compare/0.0.14...0.0.15) (2019-03-16)
+## [0.0.15](https://git.mills.io/prologic/bitcask/compare/0.0.14...0.0.15) (2019-03-16)
 
 ### Bug Fixes
 
@@ -281,11 +412,11 @@
 
 
 <a name="0.0.14"></a>
-## [0.0.14](https://github.com/prologic/bitcask/compare/0.0.13...0.0.14) (2019-03-16)
+## [0.0.14](https://git.mills.io/prologic/bitcask/compare/0.0.13...0.0.14) (2019-03-16)
 
 
 <a name="0.0.13"></a>
-## [0.0.13](https://github.com/prologic/bitcask/compare/0.0.12...0.0.13) (2019-03-16)
+## [0.0.13](https://git.mills.io/prologic/bitcask/compare/0.0.12...0.0.13) (2019-03-16)
 
 ### Features
 
@@ -293,11 +424,11 @@
 
 
 <a name="0.0.12"></a>
-## [0.0.12](https://github.com/prologic/bitcask/compare/0.0.11...0.0.12) (2019-03-14)
+## [0.0.12](https://git.mills.io/prologic/bitcask/compare/0.0.11...0.0.12) (2019-03-14)
 
 
 <a name="0.0.11"></a>
-## [0.0.11](https://github.com/prologic/bitcask/compare/0.0.10...0.0.11) (2019-03-14)
+## [0.0.11](https://git.mills.io/prologic/bitcask/compare/0.0.10...0.0.11) (2019-03-14)
 
 ### Updates
 
@@ -305,7 +436,7 @@
 
 
 <a name="0.0.10"></a>
-## [0.0.10](https://github.com/prologic/bitcask/compare/0.0.9...0.0.10) (2019-03-14)
+## [0.0.10](https://git.mills.io/prologic/bitcask/compare/0.0.9...0.0.10) (2019-03-14)
 
 ### Bug Fixes
 
@@ -318,19 +449,19 @@
 
 
 <a name="0.0.9"></a>
-## [0.0.9](https://github.com/prologic/bitcask/compare/0.0.8...0.0.9) (2019-03-14)
+## [0.0.9](https://git.mills.io/prologic/bitcask/compare/0.0.8...0.0.9) (2019-03-14)
 
 
 <a name="0.0.8"></a>
-## [0.0.8](https://github.com/prologic/bitcask/compare/0.0.7...0.0.8) (2019-03-13)
+## [0.0.8](https://git.mills.io/prologic/bitcask/compare/0.0.7...0.0.8) (2019-03-13)
 
 
 <a name="0.0.7"></a>
-## [0.0.7](https://github.com/prologic/bitcask/compare/0.0.6...0.0.7) (2019-03-13)
+## [0.0.7](https://git.mills.io/prologic/bitcask/compare/0.0.6...0.0.7) (2019-03-13)
 
 
 <a name="0.0.6"></a>
-## [0.0.6](https://github.com/prologic/bitcask/compare/0.0.5...0.0.6) (2019-03-13)
+## [0.0.6](https://git.mills.io/prologic/bitcask/compare/0.0.5...0.0.6) (2019-03-13)
 
 ### Bug Fixes
 
@@ -338,7 +469,7 @@
 
 
 <a name="0.0.5"></a>
-## [0.0.5](https://github.com/prologic/bitcask/compare/0.0.4...0.0.5) (2019-03-13)
+## [0.0.5](https://git.mills.io/prologic/bitcask/compare/0.0.4...0.0.5) (2019-03-13)
 
 ### Features
 
@@ -350,19 +481,19 @@
 
 
 <a name="0.0.4"></a>
-## [0.0.4](https://github.com/prologic/bitcask/compare/0.0.3...0.0.4) (2019-03-13)
+## [0.0.4](https://git.mills.io/prologic/bitcask/compare/0.0.3...0.0.4) (2019-03-13)
 
 ### Features
 
-* Add flock on database Open()/Close() to prevent multiple concurrent processes write access. Fixes [#2](https://github.com/prologic/bitcask/issues/2)
+* Add flock on database Open()/Close() to prevent multiple concurrent processes write access. Fixes #2
 
 
 <a name="0.0.3"></a>
-## [0.0.3](https://github.com/prologic/bitcask/compare/0.0.2...0.0.3) (2019-03-13)
+## [0.0.3](https://git.mills.io/prologic/bitcask/compare/0.0.2...0.0.3) (2019-03-13)
 
 
 <a name="0.0.2"></a>
-## [0.0.2](https://github.com/prologic/bitcask/compare/0.0.1...0.0.2) (2019-03-13)
+## [0.0.2](https://git.mills.io/prologic/bitcask/compare/0.0.1...0.0.2) (2019-03-13)
 
 
 <a name="0.0.1"></a>

Makefile

@@ -27,8 +27,14 @@ install: build
 	@go install ./cmd/bitcask/...
 	@go install ./cmd/bitcaskd/...
 
+ifeq ($(PUBLISH), 1)
 image:
-	@docker build -t prologic/bitcask .
+	@docker build --build-arg VERSION="$(VERSION)" --build-arg COMMIT="$(COMMIT)" -t prologic/bitcask .
+	@docker push prologic/bitcask
+else
+image:
+	@docker build --build-arg VERSION="$(VERSION)" --build-arg COMMIT="$(COMMIT)" -t prologic/bitcask .
+endif
 
 release:
 	@./tools/release.sh
@@ -37,7 +43,7 @@ profile: build
 	@go test -cpuprofile cpu.prof -memprofile mem.prof -v -bench .
 
 bench: build
-	@go test -v -benchmem -bench=. .
+	@go test -v -run=XXX -benchmem -bench=. .
 
 mocks:
 	@mockery -all -case underscore -output ./internal/mocks -recursive

README.md

@@ -1,42 +1,87 @@
 # bitcask
 
-![](https://github.com/prologic/bitcask/workflows/Coverage/badge.svg)
-![](https://github.com/prologic/bitcask/workflows/Docker/badge.svg)
-![](https://github.com/prologic/bitcask/workflows/Go/badge.svg)
-![](https://github.com/prologic/bitcask/workflows/ReviewDog/badge.svg)
-
-[![CodeCov](https://codecov.io/gh/prologic/bitcask/branch/master/graph/badge.svg)](https://codecov.io/gh/prologic/bitcask)
-[![Go Report Card](https://goreportcard.com/badge/prologic/bitcask)](https://goreportcard.com/report/prologic/bitcask)
-[![codebeat badge](https://codebeat.co/badges/15fba8a5-3044-4f40-936f-9e0f5d5d1fd9)](https://codebeat.co/projects/github-com-prologic-bitcask-master)
-[![GoDoc](https://godoc.org/github.com/prologic/bitcask?status.svg)](https://godoc.org/github.com/prologic/bitcask) 
-[![GitHub license](https://img.shields.io/github/license/prologic/bitcask.svg)](https://github.com/prologic/bitcask)
-[![Sourcegraph](https://sourcegraph.com/github.com/prologic/bitcask/-/badge.svg)](https://sourcegraph.com/github.com/prologic/bitcask?badge)
-[![TODOs](https://img.shields.io/endpoint?url=https://api.tickgit.com/badge?repo=github.com/prologic/bitcask)](https://www.tickgit.com/browse?repo=github.com/prologic/bitcask)
+[![Build Status](https://ci.mills.io/api/badges/prologic/bitcask/status.svg)](https://ci.mills.io/prologic/bitcask)
+[![Go Report Card](https://goreportcard.com/badge/git.mills.io/prologic/bitcask)](https://goreportcard.com/report/git.mills.io/prologic/bitcask)
+[![Go Reference](https://pkg.go.dev/badge/git.mills.io/prologic/bitcask.svg)](https://pkg.go.dev/git.mills.io/prologic/bitcask)
 
 A high performance Key/Value store written in [Go](https://golang.org) with a predictable read/write performance and high throughput. Uses a [Bitcask](https://en.wikipedia.org/wiki/Bitcask) on-disk layout (LSM+WAL) similar to [Riak](https://riak.com/)
 
-For a more feature-complete Redis-compatible server, distributed key/value store have a look at [Bitraft](https://github.com/prologic/bitraft) which uses this library as its backend. Use [Bitcask](https://github.com/prologic/bitcask) as a starting point or if you want to embed in your application, use [Bitraft](https://github.com/prologic/bitraft) if you need a complete server/client solution with high availability with a Redis-compatible API.
+For a more feature-complete Redis-compatible server, distributed key/value store have a look at [Bitraft](https://git.mills.io/prologic/bitraft) which uses this library as its backend. Use [Bitcask](https://git.mills.io/prologic/bitcask) as a starting point or if you want to embed in your application, use [Bitraft](https://git.mills.io/prologic/bitraft) if you need a complete server/client solution with high availability with a Redis-compatible API.
 
 ## Features
 
-* Embeddable (`import "github.com/prologic/bitcask"`)
+* Embedded (`import "git.mills.io/prologic/bitcask"`)
 * Builtin CLI (`bitcask`)
 * Builtin Redis-compatible server (`bitcaskd`)
 * Predictable read/write performance
 * Low latency
 * High throughput (See: [Performance](README.md#Performance) )
 
+## Is Bitcask right for my project?
+
+__NOTE__: Please read this carefully to identify whether using Bitcask is
+          suitable for your needs.
+
+`bitcask` is a **great fit** for:
+
+- Storing hundreds of thousands to millions of key/value pairs based on
+  default configuration. With the default configuration (_configurable_)
+  of 64 bytes per key and 64kB values, 1M keys would consume roughly ~600-700MB
+  of memory ~65-70GB of disk storage. These are all configurable when you
+  create a new database with `bitcask.Open(...)` with functional-style options
+  you can pass with `WithXXX()`.
+
+- As the backing store to a distributed key/value store. See for example the
+  [bitraft](https://git.mills.io/prologic/bitraft) as an example of this.
+
+- For high performance, low latency read/write workloads where you cannot fit
+  a typical hash-map into memory, but require the highest level of performance
+  and predicate read latency. Bitcask ensures only 1 read/write IOPS are ever
+  required for reading and writing key/value pairs.
+
+- As a general purpose embedded key/value store where you would have used
+  [BoltDB](https://github.com/boltdb/bolt),
+  [LevelDB](https://github.com/syndtr/goleveldb),
+  [BuntDB](https://github.com/tidwall/buntdb)
+  or similar...
+
+`bitcask` is not suited for:
+
+- Storing billions of records
+  The reason for this is the key-space is held in memory using a highly
+  performant and memory optimized adaptive radix tree thanks to
+  [go-adaptive-radix-tree](github.com/plar/go-adaptive-radix-tree) _however_
+  this means the more keys you have in your key space, the more memory is
+  consumed. Consider using a disk-backed B-Tree like [BoltDB](https://github.com/boltdb/bolt)
+  or [LevelDB](https://github.com/syndtr/goleveldb) if you intend to store a
+  large quantity of key/value pairs.
+
+> Note however that storing large amounts of data in terms of value(s) is
+> totally fine. In other wise thousands to millions of keys with large values
+> will work just fine.
+
+- Write intensive workloads. Due to the [Bitcask design](https://riak.com/assets/bitcask-intro.pdf?source=post_page---------------------------)
+  heavy write workloads that lots of key/value pairs will over time cause
+  problems like "Too many open files" (#193) errors to occur. This can be mitigated by
+  periodically compacting the data files by issuing a `.Merge()` operation however
+  if key/value pairs do not change or are never deleted, as-in only new key/value
+  pairs are ever written this will have no effect. Eventually you will run out
+  of file descriptors!
+
+> You should consider your read/write workloads carefully and ensure you set
+> appropriate file descriptor limits with `ulimit -n` that suit your needs.
+
 ## Development
 
 ```sh
-$ git clone https://github.com/prologic/bitcask.git
+$ git clone https://git.mills.io/prologic/bitcask.git
 $ make
 ```
 
 ## Install
 
 ```sh
-$ go get github.com/prologic/bitcask
+$ go get git.mills.io/prologic/bitcask
 ```
 
 ## Usage (library)
@@ -44,7 +89,7 @@ $ go get github.com/prologic/bitcask
 Install the package into your project:
 
 ```sh
-$ go get github.com/prologic/bitcask
+$ go get git.mills.io/prologic/bitcask
 ```
 
 ```go
@@ -52,7 +97,7 @@ package main
 
 import (
 	"log"
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 func main() {
@@ -64,7 +109,7 @@ func main() {
 }
 ```
 
-See the [godoc](https://godoc.org/github.com/prologic/bitcask) for further
+See the [GoDoc](https://godoc.org/git.mills.io/prologic/bitcask) for further
 documentation and other examples.
 
 ## Usage (tool)
@@ -118,14 +163,14 @@ $ docker run -d -p 6379:6379 prologic/bitcask
 
 ## Performance
 
-Benchmarks run on a 11" Macbook with a 1.4Ghz Intel Core i7:
+Benchmarks run on a 11" MacBook with a 1.4Ghz Intel Core i7:
 
 ```sh
 $ make bench
 ...
 goos: darwin
 goarch: amd64
-pkg: github.com/prologic/bitcask
+pkg: git.mills.io/prologic/bitcask
 
 BenchmarkGet/128B-4         	  316515	      3263 ns/op	  39.22 MB/s	     160 B/op	       1 allocs/op
 BenchmarkGet/256B-4         	  382551	      3204 ns/op	  79.90 MB/s	     288 B/op	       1 allocs/op
@@ -171,29 +216,25 @@ The full benchmark above shows linear performance as you increase key/value size
 
 Support the ongoing development of Bitcask!
 
-**Sponser**
+**Sponsor**
 
 - Become a [Sponsor](https://www.patreon.com/prologic)
 
-## Stargazers over time
-
-[![Stargazers over time](https://starcharts.herokuapp.com/prologic/bitcask.svg)](https://starcharts.herokuapp.com/prologic/bitcask)
-
 ## Contributors
 
-Thank you to all those that have contributed to this project, battle-tested it, used it in their own projects or products, fixed bugs, improved performance and even fix tiny typos in documentation! Thank you and keep contributing!
+Thank you to all those that have contributed to this project, battle-tested it,
+used it in their own projects or products, fixed bugs, improved performance
+and even fix tiny typos in documentation! Thank you and keep contributing!
 
-You can find an [AUTHORS](/AUTHORS) file where we keep a list of contributors to the project. If you contriibute a PR please consider adding your name there. There is also Github's own [Contributors](https://github.com/prologic/bitcask/graphs/contributors) statistics.
+You can find an [AUTHORS](/AUTHORS) file where we keep a list of contributors
+to the project. If you contribute a PR please consider adding your name there.
 
-[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/0)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/0)
-[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/1)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/1)
-[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/2)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/2)
-[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/3)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/3)
-[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/4)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/4)
-[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/5)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/5)
-[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/6)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/6)
-[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/7)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/7)
+## Related Projects
+
+- [bitraft](https://git.mills.io/prologic/bitraft) -- A Distributed Key/Value store (_using Raft_) with a Redis compatible protocol.
+- [bitcaskfs](https://git.mills.io/prologic/bitcaskfs) -- A FUSE file system for mounting a Bitcask database.
+- [bitcask-bench](https://git.mills.io/prologic/bitcask-bench) -- A benchmarking tool comparing Bitcask and several other Go key/value libraries.
 
 ## License
 
-bitcask is licensed under the term of the [MIT License](https://github.com/prologic/bitcask/blob/master/LICENSE)
+bitcask is licensed under the term of the [MIT License](https://git.mills.io/prologic/bitcask/blob/master/LICENSE)

bitcask_test.go

@@ -9,18 +9,18 @@ import (
 	"path"
 	"path/filepath"
 	"reflect"
-	"runtime"
 	"sort"
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/prologic/bitcask/internal"
-	"github.com/prologic/bitcask/internal/config"
-	"github.com/prologic/bitcask/internal/mocks"
+	"git.mills.io/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal/config"
+	"git.mills.io/prologic/bitcask/internal/mocks"
 )
 
 var (
@@ -53,12 +53,6 @@ func SortByteArrays(src [][]byte) [][]byte {
 	return sorted
 }
 
-func skipIfWindows(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip("Skipping this test on Windows")
-	}
-}
-
 func TestAll(t *testing.T) {
 	var (
 		db      *Bitcask
@@ -77,7 +71,7 @@ func TestAll(t *testing.T) {
 	})
 
 	t.Run("Put", func(t *testing.T) {
-		err = db.Put([]byte([]byte("foo")), []byte("bar"))
+		err = db.Put([]byte("foo"), []byte("bar"))
 		assert.NoError(err)
 	})
 
@@ -91,10 +85,40 @@ func TestAll(t *testing.T) {
 		assert.Equal(1, db.Len())
 	})
 
+	t.Run("PutWithTTL", func(t *testing.T) {
+		err = db.PutWithTTL([]byte("bar"), []byte("baz"), 0)
+		assert.NoError(err)
+	})
+
+	t.Run("GetExpiredKey", func(t *testing.T) {
+		time.Sleep(time.Millisecond)
+		_, err := db.Get([]byte("bar"))
+		assert.Error(err)
+		assert.Equal(ErrKeyExpired, err)
+	})
+
 	t.Run("Has", func(t *testing.T) {
 		assert.True(db.Has([]byte("foo")))
 	})
 
+	t.Run("HasWithExpired", func(t *testing.T) {
+		err = db.PutWithTTL([]byte("bar"), []byte("baz"), 0)
+		assert.NoError(err)
+		time.Sleep(time.Millisecond)
+		assert.False(db.Has([]byte("bar")))
+	})
+
+	t.Run("RunGC", func(t *testing.T) {
+		err = db.PutWithTTL([]byte("bar"), []byte("baz"), 0)
+		assert.NoError(err)
+		time.Sleep(time.Millisecond)
+		err = db.RunGC()
+		assert.NoError(err)
+		_, err := db.Get([]byte("bar"))
+		assert.Error(err)
+		assert.Equal(ErrKeyNotFound, err)
+	})
+
 	t.Run("Keys", func(t *testing.T) {
 		keys := make([][]byte, 0)
 		for key := range db.Keys() {
@@ -136,6 +160,73 @@ func TestAll(t *testing.T) {
 		assert.NoError(err)
 	})
 
+	t.Run("Backup", func(t *testing.T) {
+		path, err := ioutil.TempDir("", "backup")
+		defer os.RemoveAll(path)
+		assert.NoError(err)
+		err = db.Backup(filepath.Join(path, "db-backup"))
+		assert.NoError(err)
+	})
+
+	t.Run("Sift", func(t *testing.T) {
+		err = db.Put([]byte("toBeSifted"), []byte("siftMe"))
+		assert.NoError(err)
+		err = db.Put([]byte("notToBeSifted"), []byte("dontSiftMe"))
+		assert.NoError(err)
+		err := db.Sift(func(key []byte) (bool, error) {
+			value, err := db.Get(key)
+			if err != nil {
+				return false, err
+			}
+			if string(value) == "siftMe" {
+				return true, nil
+			}
+			return false, nil
+		})
+		assert.NoError(err)
+		_, err = db.Get([]byte("toBeSifted"))
+		assert.Equal(ErrKeyNotFound, err)
+		_, err = db.Get([]byte("notToBeSifted"))
+		assert.NoError(err)
+	})
+
+	t.Run("SiftScan", func(t *testing.T) {
+		err := db.DeleteAll()
+		assert.NoError(err)
+		err = db.Put([]byte("toBeSifted"), []byte("siftMe"))
+		assert.NoError(err)
+		err = db.Put([]byte("toBeSkipped"), []byte("siftMe"))
+		assert.NoError(err)
+		err = db.Put([]byte("toBeSiftedAsWell"), []byte("siftMe"))
+		assert.NoError(err)
+		err = db.Put([]byte("toBeSiftedButNotReally"), []byte("dontSiftMe"))
+		assert.NoError(err)
+		err = db.SiftScan([]byte("toBeSifted"), func(key []byte) (bool, error) {
+			value, err := db.Get(key)
+			if err != nil {
+				return false, err
+			}
+			if string(value) == "siftMe" {
+				return true, nil
+			}
+			return false, nil
+		})
+		assert.NoError(err)
+		_, err = db.Get([]byte("toBeSifted"))
+		assert.Equal(ErrKeyNotFound, err)
+		_, err = db.Get([]byte("toBeSiftedAsWell"))
+		assert.Equal(ErrKeyNotFound, err)
+		_, err = db.Get([]byte("toBeSkipped"))
+		assert.NoError(err)
+		_, err = db.Get([]byte("toBeSiftedButNotReally"))
+		assert.NoError(err)
+	})
+
+	t.Run("DeleteAll", func(t *testing.T) {
+		err = db.DeleteAll()
+		assert.NoError(err)
+	})
+
 	t.Run("Close", func(t *testing.T) {
 		err = db.Close()
 		assert.NoError(err)
@@ -200,6 +291,11 @@ func TestReopen(t *testing.T) {
 			assert.NoError(err)
 		})
 
+		t.Run("PutWithTTL", func(t *testing.T) {
+			err = db.PutWithTTL([]byte("bar"), []byte("baz"), 0)
+			assert.NoError(err)
+		})
+
 		t.Run("Get", func(t *testing.T) {
 			val, err := db.Get([]byte("foo"))
 			assert.NoError(err)
@@ -228,6 +324,13 @@ func TestReopen(t *testing.T) {
 			assert.Equal([]byte("foo"), val)
 		})
 
+		t.Run("GetExpiredKeyAfterReopen", func(t *testing.T) {
+			val, err := db.Get([]byte("bar"))
+			assert.Error(err)
+			assert.Equal(ErrKeyExpired, err)
+			assert.Nil(val)
+		})
+
 		t.Run("Close", func(t *testing.T) {
 			err = db.Close()
 			assert.NoError(err)
@@ -306,6 +409,64 @@ func TestDeletedKeys(t *testing.T) {
 	})
 }
 
+func TestMetadata(t *testing.T) {
+	assert := assert.New(t)
+	testdir, err := ioutil.TempDir("", "bitcask")
+	assert.NoError(err)
+	defer os.RemoveAll(testdir)
+
+	db, err := Open(testdir)
+	assert.NoError(err)
+	err = db.Put([]byte("foo"), []byte("bar"))
+	assert.NoError(err)
+	err = db.Close()
+	assert.NoError(err)
+	db, err = Open(testdir)
+	assert.NoError(err)
+
+	t.Run("IndexUptoDateAfterCloseAndOpen", func(t *testing.T) {
+		assert.Equal(true, db.metadata.IndexUpToDate)
+	})
+	t.Run("IndexUptoDateAfterPut", func(t *testing.T) {
+		assert.NoError(db.Put([]byte("foo1"), []byte("bar1")))
+		assert.Equal(false, db.metadata.IndexUpToDate)
+	})
+	t.Run("Reclaimable", func(t *testing.T) {
+		assert.Equal(int64(0), db.Reclaimable())
+	})
+	t.Run("ReclaimableAfterNewPut", func(t *testing.T) {
+		assert.NoError(db.Put([]byte("hello"), []byte("world")))
+		assert.Equal(int64(0), db.Reclaimable())
+	})
+	t.Run("ReclaimableAfterRepeatedPut", func(t *testing.T) {
+		assert.NoError(db.Put([]byte("hello"), []byte("world")))
+		assert.Equal(int64(34), db.Reclaimable())
+	})
+	t.Run("ReclaimableAfterDelete", func(t *testing.T) {
+		assert.NoError(db.Delete([]byte("hello")))
+		assert.Equal(int64(97), db.Reclaimable())
+	})
+	t.Run("ReclaimableAfterNonExistingDelete", func(t *testing.T) {
+		assert.NoError(db.Delete([]byte("hello1")))
+		assert.Equal(int64(97), db.Reclaimable())
+	})
+	t.Run("ReclaimableAfterDeleteAll", func(t *testing.T) {
+		assert.NoError(db.DeleteAll())
+		assert.Equal(int64(214), db.Reclaimable())
+	})
+	t.Run("ReclaimableAfterMerge", func(t *testing.T) {
+		assert.NoError(db.Merge())
+		assert.Equal(int64(0), db.Reclaimable())
+	})
+	t.Run("IndexUptoDateAfterMerge", func(t *testing.T) {
+		assert.Equal(true, db.metadata.IndexUpToDate)
+	})
+	t.Run("ReclaimableAfterMergeAndDeleteAll", func(t *testing.T) {
+		assert.NoError(db.DeleteAll())
+		assert.Equal(int64(0), db.Reclaimable())
+	})
+}
+
 func TestConfigErrors(t *testing.T) {
 	assert := assert.New(t)
 
@@ -337,9 +498,6 @@ func TestConfigErrors(t *testing.T) {
 }
 
 func TestAutoRecovery(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.SkipNow()
-	}
 	withAutoRecovery := []bool{false, true}
 
 	for _, autoRecovery := range withAutoRecovery {
@@ -380,6 +538,7 @@ func TestAutoRecovery(t *testing.T) {
 			require.NoError(err)
 
 			db, err = Open(testdir, WithAutoRecovery(autoRecovery))
+			t.Logf("err: %s", err)
 			require.NoError(err)
 			defer db.Close()
 			// Check that all values but the last are still intact.
@@ -415,6 +574,42 @@ func TestAutoRecovery(t *testing.T) {
 	}
 }
 
+func TestLoadIndexes(t *testing.T) {
+	assert := assert.New(t)
+	testdir, err1 := ioutil.TempDir("", "bitcask")
+	assert.NoError(err1)
+	defer os.RemoveAll(testdir)
+
+	var db *Bitcask
+	var err error
+
+	t.Run("Setup", func(t *testing.T) {
+		db, err = Open(testdir)
+		assert.NoError(err)
+		for i := 0; i < 5; i++ {
+			key := fmt.Sprintf("key%d", i)
+			val := fmt.Sprintf("val%d", i)
+			err := db.Put([]byte(key), []byte(val))
+			assert.NoError(err)
+		}
+		for i := 0; i < 5; i++ {
+			key := fmt.Sprintf("foo%d", i)
+			val := fmt.Sprintf("bar%d", i)
+			err := db.PutWithTTL([]byte(key), []byte(val), time.Duration(i)*time.Second)
+			assert.NoError(err)
+		}
+		err = db.Close()
+		assert.NoError(err)
+	})
+
+	t.Run("OpenAgain", func(t *testing.T) {
+		db, err = Open(testdir)
+		assert.NoError(err)
+		assert.Equal(10, db.trie.Size())
+		assert.Equal(5, db.ttlIndex.Size())
+	})
+}
+
 func TestReIndex(t *testing.T) {
 	assert := assert.New(t)
 
@@ -437,6 +632,16 @@ func TestReIndex(t *testing.T) {
 			assert.NoError(err)
 		})
 
+		t.Run("PutWithExpiry", func(t *testing.T) {
+			err = db.PutWithTTL([]byte("bar"), []byte("baz"), 0)
+			assert.NoError(err)
+		})
+
+		t.Run("PutWithLargeExpiry", func(t *testing.T) {
+			err = db.PutWithTTL([]byte("bar1"), []byte("baz1"), time.Hour)
+			assert.NoError(err)
+		})
+
 		t.Run("Get", func(t *testing.T) {
 			val, err := db.Get([]byte("foo"))
 			assert.NoError(err)
@@ -456,6 +661,8 @@ func TestReIndex(t *testing.T) {
 		t.Run("DeleteIndex", func(t *testing.T) {
 			err := os.Remove(filepath.Join(testdir, "index"))
 			assert.NoError(err)
+			err = os.Remove(filepath.Join(testdir, ttlIndexFile))
+			assert.NoError(err)
 		})
 	})
 
@@ -476,6 +683,16 @@ func TestReIndex(t *testing.T) {
 			assert.Equal([]byte("bar"), val)
 		})
 
+		t.Run("GetKeyWithExpiry", func(t *testing.T) {
+			val, err := db.Get([]byte("bar"))
+			assert.Error(err)
+			assert.Equal(ErrKeyExpired, err)
+			assert.Nil(val)
+			val, err = db.Get([]byte("bar1"))
+			assert.NoError(err)
+			assert.Equal([]byte("baz1"), val)
+		})
+
 		t.Run("Close", func(t *testing.T) {
 			err = db.Close()
 			assert.NoError(err)
@@ -577,6 +794,11 @@ func TestSync(t *testing.T) {
 		value := []byte("foobar")
 		err = db.Put(key, value)
 	})
+
+	t.Run("Put", func(t *testing.T) {
+		err = db.Put([]byte("hello"), []byte("world"))
+		assert.NoError(err)
+	})
 }
 
 func TestMaxKeySize(t *testing.T) {
@@ -707,8 +929,6 @@ func TestStatsError(t *testing.T) {
 	})
 
 	t.Run("Test", func(t *testing.T) {
-		skipIfWindows(t)
-
 		t.Run("FabricatedDestruction", func(t *testing.T) {
 			// This would never happen in reality :D
 			// Or would it? :)
@@ -724,8 +944,6 @@ func TestStatsError(t *testing.T) {
 }
 
 func TestDirFileModeBeforeUmask(t *testing.T) {
-	skipIfWindows(t)
-
 	assert := assert.New(t)
 
 	t.Run("Setup", func(t *testing.T) {
@@ -808,8 +1026,6 @@ func TestDirFileModeBeforeUmask(t *testing.T) {
 }
 
 func TestFileFileModeBeforeUmask(t *testing.T) {
-	skipIfWindows(t)
-
 	assert := assert.New(t)
 
 	t.Run("Setup", func(t *testing.T) {
@@ -984,7 +1200,7 @@ func TestMerge(t *testing.T) {
 
 		s3, err := db.Stats()
 		assert.NoError(err)
-		assert.Equal(1, s3.Datafiles)
+		assert.Equal(2, s3.Datafiles)
 		assert.Equal(1, s3.Keys)
 		assert.True(s3.Size > s1.Size)
 		assert.True(s3.Size < s2.Size)
@@ -1017,7 +1233,7 @@ func TestGetErrors(t *testing.T) {
 
 		mockDatafile := new(mocks.Datafile)
 		mockDatafile.On("FileID").Return(0)
-		mockDatafile.On("ReadAt", int64(0), int64(22)).Return(
+		mockDatafile.On("ReadAt", int64(0), int64(30)).Return(
 			internal.Entry{},
 			ErrMockError,
 		)
@@ -1033,7 +1249,7 @@ func TestGetErrors(t *testing.T) {
 		assert.NoError(err)
 		defer os.RemoveAll(testdir)
 
-		db, err := Open(testdir, WithMaxDatafileSize(32))
+		db, err := Open(testdir, WithMaxDatafileSize(40))
 		assert.NoError(err)
 
 		err = db.Put([]byte("foo"), []byte("bar"))
@@ -1041,7 +1257,7 @@ func TestGetErrors(t *testing.T) {
 
 		mockDatafile := new(mocks.Datafile)
 		mockDatafile.On("FileID").Return(0)
-		mockDatafile.On("ReadAt", int64(0), int64(22)).Return(
+		mockDatafile.On("ReadAt", int64(0), int64(30)).Return(
 			internal.Entry{
 				Checksum: 0x0,
 				Key:      []byte("foo"),
@@ -1208,7 +1424,7 @@ func TestCloseErrors(t *testing.T) {
 		assert.NoError(err)
 
 		mockIndexer := new(mocks.Indexer)
-		mockIndexer.On("Save", db.trie, filepath.Join(db.path, "index")).Return(ErrMockError)
+		mockIndexer.On("Save", db.trie, filepath.Join(db.path, "temp_index")).Return(ErrMockError)
 		db.indexer = mockIndexer
 
 		err = db.Close()
@@ -1279,8 +1495,6 @@ func TestMergeErrors(t *testing.T) {
 	assert := assert.New(t)
 
 	t.Run("RemoveDatabaseDirectory", func(t *testing.T) {
-		skipIfWindows(t)
-
 		testdir, err := ioutil.TempDir("", "bitcask")
 		assert.NoError(err)
 		defer os.RemoveAll(testdir)
@@ -1316,18 +1530,19 @@ func TestMergeErrors(t *testing.T) {
 		assert.NoError(err)
 		defer os.RemoveAll(testdir)
 
-		db, err := Open(testdir)
+		db, err := Open(testdir, WithMaxDatafileSize(22))
 		assert.NoError(err)
 
 		assert.NoError(db.Put([]byte("foo"), []byte("bar")))
+		assert.NoError(db.Put([]byte("bar"), []byte("baz")))
 
 		mockDatafile := new(mocks.Datafile)
-		mockDatafile.On("FileID").Return(0)
-		mockDatafile.On("ReadAt", int64(0), int64(22)).Return(
+		mockDatafile.On("Close").Return(nil)
+		mockDatafile.On("ReadAt", int64(0), int64(30)).Return(
 			internal.Entry{},
 			ErrMockError,
 		)
-		db.curr = mockDatafile
+		db.datafiles[0] = mockDatafile
 
 		err = db.Merge()
 		assert.Error(err)
@@ -1406,6 +1621,92 @@ func TestConcurrent(t *testing.T) {
 			wg.Wait()
 		})
 
+		// Test concurrent Put() with concurrent Scan()
+		t.Run("PutScan", func(t *testing.T) {
+			doPut := func(wg *sync.WaitGroup, x int) {
+				defer func() {
+					wg.Done()
+				}()
+				for i := 0; i <= 100; i++ {
+					if i%x == 0 {
+						key := []byte(fmt.Sprintf("k%d", i))
+						value := []byte(fmt.Sprintf("v%d", i))
+						err := db.Put(key, value)
+						assert.NoError(err)
+					}
+				}
+			}
+
+			doScan := func(wg *sync.WaitGroup, x int) {
+				defer func() {
+					wg.Done()
+				}()
+				for i := 0; i <= 100; i++ {
+					if i%x == 0 {
+						err := db.Scan([]byte("k"), func(key []byte) error {
+							return nil
+						})
+						assert.NoError(err)
+					}
+				}
+			}
+
+			wg := &sync.WaitGroup{}
+			wg.Add(6)
+
+			go doPut(wg, 2)
+			go doPut(wg, 3)
+			go doPut(wg, 5)
+			go doScan(wg, 1)
+			go doScan(wg, 2)
+			go doScan(wg, 4)
+
+			wg.Wait()
+		})
+
+		// XXX: This has data races
+		/* Test concurrent Scan() with concurrent Merge()
+		t.Run("ScanMerge", func(t *testing.T) {
+			doScan := func(wg *sync.WaitGroup, x int) {
+				defer func() {
+					wg.Done()
+				}()
+				for i := 0; i <= 100; i++ {
+					if i%x == 0 {
+						err := db.Scan([]byte("k"), func(key []byte) error {
+							return nil
+						})
+						assert.NoError(err)
+					}
+				}
+			}
+
+			doMerge := func(wg *sync.WaitGroup, x int) {
+				defer func() {
+					wg.Done()
+				}()
+				for i := 0; i <= 100; i++ {
+					if i%x == 0 {
+						err := db.Merge()
+						assert.NoError(err)
+					}
+				}
+			}
+
+			wg := &sync.WaitGroup{}
+			wg.Add(6)
+
+			go doScan(wg, 2)
+			go doScan(wg, 3)
+			go doScan(wg, 5)
+			go doMerge(wg, 1)
+			go doMerge(wg, 2)
+			go doMerge(wg, 4)
+
+			wg.Wait()
+		})
+		*/
+
 		t.Run("Close", func(t *testing.T) {
 			err = db.Close()
 			assert.NoError(err)
@@ -1413,6 +1714,93 @@ func TestConcurrent(t *testing.T) {
 	})
 }
 
+func TestSift(t *testing.T) {
+	assert := assert.New(t)
+
+	testdir, err := ioutil.TempDir("", "bitcask")
+	assert.NoError(err)
+
+	var db *Bitcask
+
+	t.Run("Setup", func(t *testing.T) {
+		t.Run("Open", func(t *testing.T) {
+			db, err = Open(testdir)
+			assert.NoError(err)
+		})
+
+		t.Run("Put", func(t *testing.T) {
+			var items = map[string][]byte{
+				"1":     []byte("1"),
+				"2":     []byte("2"),
+				"3":     []byte("3"),
+				"food":  []byte("pizza"),
+				"foo":   []byte([]byte("foo")),
+				"fooz":  []byte("fooz ball"),
+				"hello": []byte("world"),
+			}
+			for k, v := range items {
+				err = db.Put([]byte(k), v)
+				assert.NoError(err)
+			}
+		})
+	})
+
+	t.Run("SiftErrors", func(t *testing.T) {
+		err = db.Sift(func(key []byte) (bool, error) {
+			return false, ErrMockError
+		})
+		assert.Equal(ErrMockError, err)
+
+		err = db.SiftScan([]byte("fo"), func(key []byte) (bool, error) {
+			return true, ErrMockError
+		})
+		assert.Equal(ErrMockError, err)
+	})
+}
+func TestSiftScan(t *testing.T) {
+	assert := assert.New(t)
+
+	testdir, err := ioutil.TempDir("", "bitcask")
+	assert.NoError(err)
+
+	var db *Bitcask
+
+	t.Run("Setup", func(t *testing.T) {
+		t.Run("Open", func(t *testing.T) {
+			db, err = Open(testdir)
+			assert.NoError(err)
+		})
+
+		t.Run("Put", func(t *testing.T) {
+			var items = map[string][]byte{
+				"1":     []byte("1"),
+				"2":     []byte("2"),
+				"3":     []byte("3"),
+				"food":  []byte("pizza"),
+				"foo":   []byte([]byte("foo")),
+				"fooz":  []byte("fooz ball"),
+				"hello": []byte("world"),
+			}
+			for k, v := range items {
+				err = db.Put([]byte(k), v)
+				assert.NoError(err)
+			}
+		})
+	})
+
+	t.Run("SiftScanErrors", func(t *testing.T) {
+		err = db.SiftScan([]byte("fo"), func(key []byte) (bool, error) {
+			return false, ErrMockError
+		})
+		assert.Equal(ErrMockError, err)
+
+		err = db.SiftScan([]byte("fo"), func(key []byte) (bool, error) {
+			return true, ErrMockError
+		})
+		assert.Equal(ErrMockError, err)
+	})
+}
+
 func TestScan(t *testing.T) {
 	assert := assert.New(t)
 
@@ -1472,6 +1860,146 @@ func TestScan(t *testing.T) {
 		assert.Equal(ErrMockError, err)
 	})
 }
+func TestSiftRange(t *testing.T) {
+	assert := assert.New(t)
+
+	testdir, err := ioutil.TempDir("", "bitcask")
+	assert.NoError(err)
+
+	var db *Bitcask
+
+	t.Run("Setup", func(t *testing.T) {
+		t.Run("Open", func(t *testing.T) {
+			db, err = Open(testdir)
+			assert.NoError(err)
+		})
+
+		t.Run("Put", func(t *testing.T) {
+			for i := 1; i < 10; i++ {
+				key := []byte(fmt.Sprintf("foo_%d", i))
+				val := []byte(fmt.Sprintf("%d", i))
+				err = db.Put(key, val)
+				assert.NoError(err)
+			}
+		})
+	})
+
+	t.Run("SiftRange", func(t *testing.T) {
+		var (
+			vals     [][]byte
+			expected = [][]byte{
+				[]byte("1"),
+				[]byte("2"),
+				[]byte("4"),
+				[]byte("5"),
+				[]byte("6"),
+				[]byte("7"),
+				[]byte("8"),
+				[]byte("9"),
+			}
+		)
+
+		err = db.SiftRange([]byte("foo_3"), []byte("foo_7"), func(key []byte) (bool, error) {
+			val, err := db.Get(key)
+			assert.NoError(err)
+			if string(val) == "3" {
+				return true, nil
+			}
+			return false, nil
+		})
+		err = db.Fold(func(key []byte) error {
+			val, err := db.Get(key)
+			assert.NoError(err)
+			vals = append(vals, val)
+
+			return nil
+		})
+
+		_, err = db.Get([]byte("foo_3"))
+		assert.Equal(ErrKeyNotFound, err)
+		vals = SortByteArrays(vals)
+		assert.Equal(expected, vals)
+	})
+
+	t.Run("SiftRangeErrors", func(t *testing.T) {
+		err = db.SiftRange([]byte("foo_3"), []byte("foo_7"), func(key []byte) (bool, error) {
+			return true, ErrMockError
+		})
+		assert.Error(err)
+		assert.Equal(ErrMockError, err)
+	})
+
+	t.Run("InvalidRange", func(t *testing.T) {
+		err = db.SiftRange([]byte("foo_3"), []byte("foo_1"), func(key []byte) (bool, error) {
+			return false, nil
+		})
+		assert.Error(err)
+		assert.Equal(ErrInvalidRange, err)
+	})
+}
+
+func TestRange(t *testing.T) {
+	assert := assert.New(t)
+
+	testdir, err := ioutil.TempDir("", "bitcask")
+	assert.NoError(err)
+
+	var db *Bitcask
+
+	t.Run("Setup", func(t *testing.T) {
+		t.Run("Open", func(t *testing.T) {
+			db, err = Open(testdir)
+			assert.NoError(err)
+		})
+
+		t.Run("Put", func(t *testing.T) {
+			for i := 1; i < 10; i++ {
+				key := []byte(fmt.Sprintf("foo_%d", i))
+				val := []byte(fmt.Sprintf("%d", i))
+				err = db.Put(key, val)
+				assert.NoError(err)
+			}
+		})
+	})
+
+	t.Run("Range", func(t *testing.T) {
+		var (
+			vals     [][]byte
+			expected = [][]byte{
+				[]byte("3"),
+				[]byte("4"),
+				[]byte("5"),
+				[]byte("6"),
+				[]byte("7"),
+			}
+		)
+
+		err = db.Range([]byte("foo_3"), []byte("foo_7"), func(key []byte) error {
+			val, err := db.Get(key)
+			assert.NoError(err)
+			vals = append(vals, val)
+			return nil
+		})
+		vals = SortByteArrays(vals)
+		assert.Equal(expected, vals)
+	})
+
+	t.Run("RangeErrors", func(t *testing.T) {
+		err = db.Range([]byte("foo_3"), []byte("foo_7"), func(key []byte) error {
+			return ErrMockError
+		})
+		assert.Error(err)
+		assert.Equal(ErrMockError, err)
+	})
+
+	t.Run("InvalidRange", func(t *testing.T) {
+		err = db.Range([]byte("foo_3"), []byte("foo_1"), func(key []byte) error {
+			return nil
+		})
+		assert.Error(err)
+		assert.Equal(ErrInvalidRange, err)
+	})
+}
 
 func TestLocking(t *testing.T) {
 	assert := assert.New(t)
@@ -1485,7 +2013,95 @@ func TestLocking(t *testing.T) {
 
 	_, err = Open(testdir)
 	assert.Error(err)
-	assert.Equal(ErrDatabaseLocked, err)
+}
+
+func TestLockingAfterMerge(t *testing.T) {
+	assert := assert.New(t)
+
+	testdir, err := ioutil.TempDir("", "bitcask")
+	assert.NoError(err)
+
+	db, err := Open(testdir)
+	assert.NoError(err)
+	defer db.Close()
+
+	_, err = Open(testdir)
+	assert.Error(err)
+
+	err = db.Merge()
+	assert.NoError(err)
+
+	// This should still error.
+	_, err = Open(testdir)
+	assert.Error(err)
+}
+
+func TestGetExpiredInsideFold(t *testing.T) {
+	assert := assert.New(t)
+
+	testdir, err := ioutil.TempDir("", "bitcask")
+	assert.NoError(err)
+
+	db, err := Open(testdir)
+	assert.NoError(err)
+	defer db.Close()
+	// Add a node to the tree that won't expire
+	db.Put([]byte("static"), []byte("static"))
+	// Add a node that expires almost immediately to the tree
+	db.PutWithTTL([]byte("shortLived"), []byte("shortLived"), 1*time.Millisecond)
+	db.Put([]byte("skipped"), []byte("skipped"))
+	db.Put([]byte("static2"), []byte("static2"))
+	time.Sleep(2 * time.Millisecond)
+	var arr []string
+	_ = db.Fold(func(key []byte) error {
+		val, err := db.Get(key)
+		switch string(key) {
+		case "skipped":
+			fallthrough
+		case "static2":
+			fallthrough
+		case "static":
+			assert.NoError(err)
+			assert.Equal(string(val), string(key))
+		case "shortLived":
+			assert.Error(err)
+		}
+		arr = append(arr, string(val))
+		return nil
+	})
+	assert.Contains(arr, "skipped")
+}
+
+func TestRunGCDeletesAllExpired(t *testing.T) {
+	assert := assert.New(t)
+
+	testdir, err := ioutil.TempDir("", "bitcask")
+	assert.NoError(err)
+
+	db, err := Open(testdir)
+	assert.NoError(err)
+	defer db.Close()
+
+	// Add a node to the tree that won't expire
+	db.Put([]byte("static"), []byte("static"))
+
+	// Add a node that expires almost immediately to the tree
+	db.PutWithTTL([]byte("shortLived"), []byte("shortLived"), 0)
+	db.PutWithTTL([]byte("longLived"), []byte("longLived"), time.Hour)
+	db.PutWithTTL([]byte("longLived2"), []byte("longLived2"), time.Hour)
+	db.PutWithTTL([]byte("shortLived2"), []byte("shortLived2"), 0)
+	db.PutWithTTL([]byte("shortLived3"), []byte("shortLived3"), 0)
+	db.Put([]byte("static2"), []byte("static2"))
+
+	// Sleep a bit and run the Garbage Collector
+	time.Sleep(3 * time.Millisecond)
+	db.RunGC()
+
+	_ = db.Fold(func(key []byte) error {
+		_, err := db.Get(key)
+		assert.NoError(err)
+		return nil
+	})
 }
 
 type benchmarkTestCase struct {

cmd/bitcask/del.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var delCmd = &cobra.Command{

cmd/bitcask/export.go

@@ -11,7 +11,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var errNotAllDataWritten = errors.New("error: not all data written")

cmd/bitcask/get.go

@@ -8,7 +8,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var getCmd = &cobra.Command{

cmd/bitcask/import.go

@@ -11,7 +11,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var importCmd = &cobra.Command{

cmd/bitcask/initdb.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var initdbCmd = &cobra.Command{

cmd/bitcask/keys.go

@@ -8,7 +8,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var keysCmd = &cobra.Command{

cmd/bitcask/merge.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var mergeCmd = &cobra.Command{

cmd/bitcask/put.go

@@ -10,7 +10,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var putCmd = &cobra.Command{

cmd/bitcask/range.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"git.mills.io/prologic/bitcask"
+)
+
+var rangeCmd = &cobra.Command{
+	Use:     "range <start> <end>",
+	Aliases: []string{},
+	Short:   "Perform a range scan for keys from a start to end key",
+	Long: `This performa a range scan for keys  starting with the given start
+key and ending with the end key. This uses a Trie to search for matching keys
+within the range and returns all matched keys.`,
+	Args: cobra.ExactArgs(2),
+	Run: func(cmd *cobra.Command, args []string) {
+		path := viper.GetString("path")
+
+		start := args[0]
+		end := args[1]
+
+		os.Exit(_range(path, start, end))
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(rangeCmd)
+}
+
+func _range(path, start, end string) int {
+	db, err := bitcask.Open(path)
+	if err != nil {
+		log.WithError(err).Error("error opening database")
+		return 1
+	}
+	defer db.Close()
+
+	err = db.Range([]byte(start), []byte(end), func(key []byte) error {
+		value, err := db.Get(key)
+		if err != nil {
+			log.WithError(err).Error("error reading key")
+			return err
+		}
+
+		fmt.Printf("%s\n", string(value))
+		log.WithField("key", key).WithField("value", value).Debug("key/value")
+		return nil
+	})
+	if err != nil {
+		log.WithError(err).Error("error ranging over keys")
+		return 1
+	}
+
+	return 0
+}

cmd/bitcask/recover.go

@@ -6,11 +6,11 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/prologic/bitcask"
-	"github.com/prologic/bitcask/internal"
-	"github.com/prologic/bitcask/internal/config"
-	"github.com/prologic/bitcask/internal/data/codec"
-	"github.com/prologic/bitcask/internal/index"
+	"git.mills.io/prologic/bitcask"
+	"git.mills.io/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal/config"
+	"git.mills.io/prologic/bitcask/internal/data/codec"
+	"git.mills.io/prologic/bitcask/internal/index"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"

cmd/bitcask/root.go

@@ -8,7 +8,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 )
 
 // RootCmd represents the base command when called without any subcommands

cmd/bitcask/scan.go

@@ -8,7 +8,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var scanCmd = &cobra.Command{

cmd/bitcask/stats.go

@@ -9,7 +9,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 var statsCmd = &cobra.Command{

cmd/bitcaskd/main.go

@@ -7,7 +7,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	flag "github.com/spf13/pflag"
 
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 )
 
 var (

cmd/bitcaskd/server.go

@@ -1,16 +1,18 @@
 package main
 
 import (
+	"encoding/binary"
 	"fmt"
 	"os"
 	"os/signal"
 	"strings"
 	"syscall"
+	"time"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/tidwall/redcon"
 
-	"github.com/prologic/bitcask"
+	"git.mills.io/prologic/bitcask"
 )
 
 type server struct {
@@ -32,25 +34,37 @@ func newServer(bind, dbpath string) (*server, error) {
 }
 
 func (s *server) handleSet(cmd redcon.Command, conn redcon.Conn) {
-	if len(cmd.Args) != 3 {
+	if len(cmd.Args) != 3 && len(cmd.Args) != 4 {
 		conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
 		return
 	}
+
+	var ttl *time.Duration
+
 	key := cmd.Args[1]
 	value := cmd.Args[2]
 
-	err := s.db.Lock()
-	if err != nil {
-		conn.WriteError("ERR " + fmt.Errorf("failed to lock db: %v", err).Error() + "")
-		return
+	if len(cmd.Args) == 4 {
+		val, n := binary.Varint(cmd.Args[3])
+		if n <= 0 {
+			conn.WriteError("ERR error parsing ttl")
+			return
+		}
+		d := time.Duration(val) * time.Millisecond
+		ttl = &d
 	}
-	defer s.db.Unlock()
 
-	if err := s.db.Put(key, value); err != nil {
-		conn.WriteString(fmt.Sprintf("ERR: %s", err))
+	if ttl != nil {
+		if err := s.db.PutWithTTL(key, value, *ttl); err != nil {
+			conn.WriteString(fmt.Sprintf("ERR: %s", err))
+		}
 	} else {
-		conn.WriteString("OK")
+		if err := s.db.Put(key, value); err != nil {
+			conn.WriteString(fmt.Sprintf("ERR: %s", err))
+		}
 	}
+
+	conn.WriteString("OK")
 }
 
 func (s *server) handleGet(cmd redcon.Command, conn redcon.Conn) {
@@ -61,13 +75,6 @@ func (s *server) handleGet(cmd redcon.Command, conn redcon.Conn) {
 
 	key := cmd.Args[1]
 
-	err := s.db.Lock()
-	if err != nil {
-		conn.WriteError("ERR " + fmt.Errorf("failed to lock db: %v", err).Error() + "")
-		return
-	}
-	defer s.db.Unlock()
-
 	value, err := s.db.Get(key)
 	if err != nil {
 		conn.WriteNull()
@@ -77,17 +84,41 @@ func (s *server) handleGet(cmd redcon.Command, conn redcon.Conn) {
 }
 
 func (s *server) handleKeys(cmd redcon.Command, conn redcon.Conn) {
-	err := s.db.Lock()
-	if err != nil {
-		conn.WriteError("ERR " + fmt.Errorf("failed to lock db: %v", err).Error() + "")
+	if len(cmd.Args) != 2 {
+		conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
 		return
 	}
-	defer s.db.Unlock()
 
-	conn.WriteArray(s.db.Len())
-	for key := range s.db.Keys() {
-		conn.WriteBulk(key)
+	pattern := string(cmd.Args[1])
+
+	// Fast-track condition for improved speed
+	if pattern == "*" {
+		conn.WriteArray(s.db.Len())
+		for key := range s.db.Keys() {
+			conn.WriteBulk(key)
+		}
+		return
 	}
+
+	// Prefix handling
+	if strings.Count(pattern, "*") == 1 && strings.HasSuffix(pattern, "*") {
+		prefix := strings.ReplaceAll(pattern, "*", "")
+		count := 0
+		keys := make([][]byte, 0)
+		s.db.Scan([]byte(prefix), func(key []byte) error {
+			keys = append(keys, key)
+			count++
+			return nil
+		})
+		conn.WriteArray(count)
+		for _, key := range keys {
+			conn.WriteBulk(key)
+		}
+		return
+	}
+
+	// No results means empty array
+	conn.WriteArray(0)
 }
 
 func (s *server) handleExists(cmd redcon.Command, conn redcon.Conn) {
@@ -98,13 +129,6 @@ func (s *server) handleExists(cmd redcon.Command, conn redcon.Conn) {
 
 	key := cmd.Args[1]
 
-	err := s.db.Lock()
-	if err != nil {
-		conn.WriteError("ERR " + fmt.Errorf("failed to lock db: %v", err).Error() + "")
-		return
-	}
-	defer s.db.Unlock()
-
 	if s.db.Has(key) {
 		conn.WriteInt(1)
 	} else {
@@ -120,13 +144,6 @@ func (s *server) handleDel(cmd redcon.Command, conn redcon.Conn) {
 
 	key := cmd.Args[1]
 
-	err := s.db.Lock()
-	if err != nil {
-		conn.WriteError("ERR " + fmt.Errorf("failed to lock db: %v", err).Error() + "")
-		return
-	}
-	defer s.db.Unlock()
-
 	if err := s.db.Delete(key); err != nil {
 		conn.WriteInt(0)
 	} else {

cmd/bitcaskd/server_test.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"net"
+	"strconv"
+	"testing"
+
+	"github.com/tidwall/redcon"
+)
+
+func TestHandleKeys(t *testing.T) {
+	s, err := newServer(":61234", "./test.db")
+	if err != nil {
+		t.Fatalf("Unable to create server: %v", err)
+	}
+	s.db.Put([]byte("foo"), []byte("bar"))
+	testCases := []TestCase{
+		{
+			Command: redcon.Command{
+				Raw:  []byte("KEYS *"),
+				Args: [][]byte{[]byte("KEYS"), []byte("*")},
+			},
+			Expected: "1,foo",
+		},
+		{
+			Command: redcon.Command{
+				Raw:  []byte("KEYS fo*"),
+				Args: [][]byte{[]byte("KEYS"), []byte("fo*")},
+			},
+			Expected: "1,foo",
+		},
+		{
+			Command: redcon.Command{
+				Raw:  []byte("KEYS ba*"),
+				Args: [][]byte{[]byte("KEYS"), []byte("ba*")},
+			},
+			Expected: "0",
+		},
+		{
+			Command: redcon.Command{
+				Raw:  []byte("KEYS *oo"),
+				Args: [][]byte{[]byte("KEYS"), []byte("*oo")},
+			},
+			Expected: "0",
+		},
+	}
+	for _, testCase := range testCases {
+		conn := DummyConn{}
+		s.handleKeys(testCase.Command, &conn)
+		if testCase.Expected != conn.Result {
+			t.Fatalf("s.handleKeys failed: expected '%s', got '%s'", testCase.Expected, conn.Result)
+		}
+	}
+}
+
+type TestCase struct {
+	Command  redcon.Command
+	Expected string
+}
+
+type DummyConn struct {
+	Result string
+}
+
+func (dc *DummyConn) RemoteAddr() string {
+	return ""
+}
+func (dc *DummyConn) Close() error {
+	return nil
+}
+func (dc *DummyConn) WriteError(msg string)  {}
+func (dc *DummyConn) WriteString(str string) {}
+func (dc *DummyConn) WriteBulk(bulk []byte) {
+	dc.Result += "," + string(bulk)
+}
+func (dc *DummyConn) WriteBulkString(bulk string) {}
+func (dc *DummyConn) WriteInt(num int)            {}
+func (dc *DummyConn) WriteInt64(num int64)        {}
+func (dc *DummyConn) WriteUint64(num uint64)      {}
+func (dc *DummyConn) WriteArray(count int) {
+	dc.Result = strconv.Itoa(count)
+}
+func (dc *DummyConn) WriteNull()               {}
+func (dc *DummyConn) WriteRaw(data []byte)     {}
+func (dc *DummyConn) WriteAny(any interface{}) {}
+func (dc *DummyConn) Context() interface{} {
+	return nil
+}
+func (dc *DummyConn) SetContext(v interface{}) {}
+func (dc *DummyConn) SetReadBuffer(bytes int)  {}
+func (dc *DummyConn) Detach() redcon.DetachedConn {
+	return nil
+}
+func (dc *DummyConn) ReadPipeline() []redcon.Command {
+	return nil
+}
+func (dc *DummyConn) PeekPipeline() []redcon.Command {
+	return nil
+}
+func (dc *DummyConn) NetConn() net.Conn {
+	return nil
+}

errors.go

@@ -0,0 +1,77 @@
+package bitcask
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	// ErrKeyNotFound is the error returned when a key is not found
+	ErrKeyNotFound = errors.New("error: key not found")
+
+	// ErrKeyTooLarge is the error returned for a key that exceeds the
+	// maximum allowed key size (configured with WithMaxKeySize).
+	ErrKeyTooLarge = errors.New("error: key too large")
+
+	// ErrKeyExpired is the error returned when a key is queried which has
+	// already expired (due to ttl)
+	ErrKeyExpired = errors.New("error: key expired")
+
+	// ErrEmptyKey is the error returned for a value with an empty key.
+	ErrEmptyKey = errors.New("error: empty key")
+
+	// ErrValueTooLarge is the error returned for a value that exceeds the
+	// maximum allowed value size (configured with WithMaxValueSize).
+	ErrValueTooLarge = errors.New("error: value too large")
+
+	// ErrChecksumFailed is the error returned if a key/value retrieved does
+	// not match its CRC checksum
+	ErrChecksumFailed = errors.New("error: checksum failed")
+
+	// ErrDatabaseLocked is the error returned if the database is locked
+	// (typically opened by another process)
+	ErrDatabaseLocked = errors.New("error: database locked")
+
+	// ErrInvalidRange is the error returned when the range scan is invalid
+	ErrInvalidRange = errors.New("error: invalid range")
+
+	// ErrInvalidVersion is the error returned when the database version is invalid
+	ErrInvalidVersion = errors.New("error: invalid db version")
+
+	// ErrMergeInProgress is the error returned if merge is called when already a merge
+	// is in progress
+	ErrMergeInProgress = errors.New("error: merge already in progress")
+)
+
+// ErrBadConfig is the error returned on failure to load the database config
+type ErrBadConfig struct {
+	Err error
+}
+
+func (e *ErrBadConfig) Is(target error) bool {
+	if _, ok := target.(*ErrBadConfig); ok {
+		return true
+	}
+	return errors.Is(e.Err, target)
+}
+func (e *ErrBadConfig) Unwrap() error { return e.Err }
+func (e *ErrBadConfig) Error() string {
+	return fmt.Sprintf("error reading config.json: %s", e.Err)
+}
+
+// ErrBadMetadata is the error returned on failure to load the database metadata
+type ErrBadMetadata struct {
+	Err error
+}
+
+func (e *ErrBadMetadata) Is(target error) bool {
+	if _, ok := target.(*ErrBadMetadata); ok {
+		return true
+	}
+	return errors.Is(e.Err, target)
+}
+
+func (e *ErrBadMetadata) Unwrap() error { return e.Err }
+func (e *ErrBadMetadata) Error() string {
+	return fmt.Sprintf("error reading meta.json: %s", e.Err)
+}

go.mod

@@ -1,24 +1,18 @@
-module github.com/prologic/bitcask
+module git.mills.io/prologic/bitcask
 
-go 1.13
+go 1.16
 
 require (
+	github.com/abcum/lcp v0.0.0-20201209214815-7a3f3840be81
 	github.com/gofrs/flock v0.8.0
-	github.com/pelletier/go-toml v1.6.0 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/plar/go-adaptive-radix-tree v1.0.4
-	github.com/sirupsen/logrus v1.7.0
-	github.com/spf13/afero v1.2.2 // indirect
-	github.com/spf13/cast v1.3.1 // indirect
+	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v0.0.7
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.7.1
+	github.com/spf13/viper v1.8.1
 	github.com/stretchr/objx v0.2.0 // indirect
-	github.com/stretchr/testify v1.6.1
-	github.com/tidwall/redcon v1.4.0
+	github.com/stretchr/testify v1.7.0
+	github.com/tidwall/redcon v1.4.1
 	golang.org/x/exp v0.0.0-20200228211341-fcea875c7e85
-	golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect
-	gopkg.in/ini.v1 v1.53.0 // indirect
-	gopkg.in/yaml.v2 v2.2.8 // indirect
 )

go.sum

@@ -5,18 +5,46 @@ cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6A
 cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
 cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
 cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
+cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
+cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
+cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
+cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
+cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
+cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
+cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
+cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
+cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
+cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
+cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
+cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
+cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
+cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
+cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
+cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
+cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
+cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
+cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
+cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/abcum/lcp v0.0.0-20201209214815-7a3f3840be81 h1:uHogIJ9bXH75ZYrXnVShHIyywFiUZ7OOabwd9Sfd8rw=
+github.com/abcum/lcp v0.0.0-20201209214815-7a3f3840be81/go.mod h1:6ZvnjTZX1LNo1oLpfaJK8h+MXqHxcBFBIwkgsv+xlv0=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -24,15 +52,22 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -40,45 +75,101 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/flock v0.8.0 h1:MSdYClljsF3PbENUUEx85nkWfJSGfzYI9yEBZOJz6CY=
 github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
+github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -100,27 +191,31 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
-github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
+github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@@ -132,21 +227,23 @@ github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eI
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
+github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=
-github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys=
+github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ=
+github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/plar/go-adaptive-radix-tree v1.0.4 h1:Ucd8R6RH2E7RW8ZtDKrsWyOD3paG2qqJO0I20WQ8oWQ=
 github.com/plar/go-adaptive-radix-tree v1.0.4/go.mod h1:Ot8d28EII3i7Lv4PSvBlF8ejiD/CtRYDuPsySJbSaK8=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -156,87 +253,109 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
-github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY=
+github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.7 h1:FfTH+vuMXOas8jmfb5/M7dzEYx7LpcLb7a0LPe34uOU=
 github.com/spf13/cobra v0.0.7/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
-github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
-github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
-github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44=
+github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/tidwall/btree v0.2.2 h1:VVo0JW/tdidNdQzNsDR4wMbL3heaxA1DGleyzQ3/niY=
-github.com/tidwall/btree v0.2.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
-github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
-github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
-github.com/tidwall/redcon v1.4.0 h1:y2PmDD55STRdy4S98qP/Dn+gZG+cPVvIDi9BJV2aOwA=
-github.com/tidwall/redcon v1.4.0/go.mod h1:IGzxyoKE3Ea5AWIXo/ZHP+hzY8sWXaMKr7KlFgcWSZU=
+github.com/tidwall/btree v0.4.2 h1:aLwwJlG+InuFzdAPuBf9YCAR1LvSQ9zhC5aorFPlIPs=
+github.com/tidwall/btree v0.4.2/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8=
+github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE=
+github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/redcon v1.4.1 h1:oupK+lM1FeSGNhhZn85KvofEpboQReM1eIKNWTmD3K8=
+github.com/tidwall/redcon v1.4.1/go.mod h1:XwNPFbJ4ShWNNSA2Jazhbdje6jegTCcwFR6mfaADvHA=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
+go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
+go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
 golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
 golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
+golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/exp v0.0.0-20200228211341-fcea875c7e85 h1:jqhIzSw5SQNkbu5hOGpgMHhkfXxrbsLJdkIRcX19gCY=
 golang.org/x/exp v0.0.0-20200228211341-fcea875c7e85/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
@@ -248,12 +367,22 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -269,14 +398,53 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -290,17 +458,51 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -311,6 +513,7 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
@@ -318,18 +521,73 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
+golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
+google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
+google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
+google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
+google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
+google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
+google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
+google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -339,32 +597,96 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
 google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
+google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
+google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
+google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
+google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
-gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/ini.v1 v1.53.0 h1:c7ruDvTQi0MUTFuNpDRXLSjs7xT4TerM1icIg4uKWRg=
-gopkg.in/ini.v1 v1.53.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=
+gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

internal/config/config.go

@@ -13,6 +13,7 @@ type Config struct {
 	MaxValueSize            uint64 `json:"max_value_size"`
 	Sync                    bool   `json:"sync"`
 	AutoRecovery            bool   `json:"autorecovery"`
+	DBVersion               uint32 `json:"db_version"`
 	DirFileModeBeforeUmask  os.FileMode
 	FileFileModeBeforeUmask os.FileMode
 }

internal/data/codec/decoder.go

@@ -3,9 +3,10 @@ package codec
 import (
 	"encoding/binary"
 	"io"
+	"time"
 
 	"github.com/pkg/errors"
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 )
 
 var (
@@ -49,13 +50,13 @@ func (d *Decoder) Decode(v *internal.Entry) (int64, error) {
 		return 0, err
 	}
 
-	buf := make([]byte, uint64(actualKeySize)+actualValueSize+checksumSize)
+	buf := make([]byte, uint64(actualKeySize)+actualValueSize+checksumSize+ttlSize)
 	if _, err = io.ReadFull(d.r, buf); err != nil {
 		return 0, errTruncatedData
 	}
 
 	decodeWithoutPrefix(buf, actualKeySize, v)
-	return int64(keySize + valueSize + uint64(actualKeySize) + actualValueSize + checksumSize), nil
+	return int64(keySize + valueSize + uint64(actualKeySize) + actualValueSize + checksumSize + ttlSize), nil
 }
 
 // DecodeEntry decodes a serialized entry
@@ -74,7 +75,7 @@ func getKeyValueSizes(buf []byte, maxKeySize uint32, maxValueSize uint64) (uint3
 	actualKeySize := binary.BigEndian.Uint32(buf[:keySize])
 	actualValueSize := binary.BigEndian.Uint64(buf[keySize:])
 
-	if actualKeySize > maxKeySize || actualValueSize > maxValueSize || actualKeySize == 0 {
+	if (maxKeySize > 0 && actualKeySize > maxKeySize) || (maxValueSize > 0 && actualValueSize > maxValueSize) || actualKeySize == 0 {
 
 		return 0, 0, errInvalidKeyOrValueSize
 	}
@@ -84,8 +85,18 @@ func getKeyValueSizes(buf []byte, maxKeySize uint32, maxValueSize uint64) (uint3
 
 func decodeWithoutPrefix(buf []byte, valueOffset uint32, v *internal.Entry) {
 	v.Key = buf[:valueOffset]
-	v.Value = buf[valueOffset : len(buf)-checksumSize]
-	v.Checksum = binary.BigEndian.Uint32(buf[len(buf)-checksumSize:])
+	v.Value = buf[valueOffset : len(buf)-checksumSize-ttlSize]
+	v.Checksum = binary.BigEndian.Uint32(buf[len(buf)-checksumSize-ttlSize : len(buf)-ttlSize])
+	v.Expiry = getKeyExpiry(buf)
+}
+
+func getKeyExpiry(buf []byte) *time.Time {
+	expiry := binary.BigEndian.Uint64(buf[len(buf)-ttlSize:])
+	if expiry == uint64(0) {
+		return nil
+	}
+	t := time.Unix(int64(expiry), 0).UTC()
+	return &t
 }
 
 // IsCorruptedData indicates if the error correspondes to possible data corruption

internal/data/codec/decoder_test.go

@@ -5,8 +5,9 @@ import (
 	"encoding/binary"
 	"io"
 	"testing"
+	"time"
 
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -107,3 +108,23 @@ func TestTruncatedData(t *testing.T) {
 		})
 	}
 }
+
+func TestDecodeWithoutPrefix(t *testing.T) {
+	assert := assert.New(t)
+	e := internal.Entry{}
+	buf := []byte{0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 7, 109, 121, 107, 101, 121, 109, 121, 118, 97, 108, 117, 101, 0, 6, 81, 189, 0, 0, 0, 0, 95, 117, 28, 0}
+	valueOffset := uint32(5)
+	mockTime := time.Date(2020, 10, 1, 0, 0, 0, 0, time.UTC)
+	expectedEntry := internal.Entry{
+		Key:      []byte("mykey"),
+		Value:    []byte("myvalue"),
+		Checksum: 414141,
+		Expiry:   &mockTime,
+	}
+	decodeWithoutPrefix(buf[keySize+valueSize:], valueOffset, &e)
+	assert.Equal(expectedEntry.Key, e.Key)
+	assert.Equal(expectedEntry.Value, e.Value)
+	assert.Equal(expectedEntry.Checksum, e.Checksum)
+	assert.Equal(expectedEntry.Offset, e.Offset)
+	assert.Equal(*expectedEntry.Expiry, *e.Expiry)
+}

internal/data/codec/encoder.go

@@ -6,13 +6,15 @@ import (
 	"io"
 
 	"github.com/pkg/errors"
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 )
 
 const (
 	keySize      = 4
 	valueSize    = 8
 	checksumSize = 4
+	ttlSize      = 8
+	MetaInfoSize = keySize + valueSize + checksumSize + ttlSize
 )
 
 // NewEncoder creates a streaming Entry encoder.
@@ -49,9 +51,19 @@ func (e *Encoder) Encode(msg internal.Entry) (int64, error) {
 		return 0, errors.Wrap(err, "failed writing checksum data")
 	}
 
+	bufTTL := bufKeyValue[:ttlSize]
+	if msg.Expiry == nil {
+		binary.BigEndian.PutUint64(bufTTL, uint64(0))
+	} else {
+		binary.BigEndian.PutUint64(bufTTL, uint64(msg.Expiry.Unix()))
+	}
+	if _, err := e.w.Write(bufTTL); err != nil {
+		return 0, errors.Wrap(err, "failed writing ttl data")
+	}
+
 	if err := e.w.Flush(); err != nil {
 		return 0, errors.Wrap(err, "failed flushing data")
 	}
 
-	return int64(keySize + valueSize + len(msg.Key) + len(msg.Value) + checksumSize), nil
+	return int64(keySize + valueSize + len(msg.Key) + len(msg.Value) + checksumSize + ttlSize), nil
 }

internal/data/codec/encoder_test.go

@@ -4,8 +4,9 @@ import (
 	"bytes"
 	"encoding/hex"
 	"testing"
+	"time"
 
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -14,15 +15,17 @@ func TestEncode(t *testing.T) {
 	assert := assert.New(t)
 
 	var buf bytes.Buffer
+	mockTime := time.Date(2020, 10, 1, 0, 0, 0, 0, time.UTC)
 	encoder := NewEncoder(&buf)
 	_, err := encoder.Encode(internal.Entry{
 		Key:      []byte("mykey"),
 		Value:    []byte("myvalue"),
 		Checksum: 414141,
 		Offset:   424242,
+		Expiry:   &mockTime,
 	})
 
-	expectedHex := "0000000500000000000000076d796b65796d7976616c7565000651bd"
+	expectedHex := "0000000500000000000000076d796b65796d7976616c7565000651bd000000005f751c00"
 	if assert.NoError(err) {
 		assert.Equal(expectedHex, hex.EncodeToString(buf.Bytes()))
 	}

internal/data/datafile.go

@@ -6,9 +6,9 @@ import (
 	"path/filepath"
 	"sync"
 
+	"git.mills.io/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal/data/codec"
 	"github.com/pkg/errors"
-	"github.com/prologic/bitcask/internal"
-	"github.com/prologic/bitcask/internal/data/codec"
 	"golang.org/x/exp/mmap"
 )
 
@@ -74,9 +74,11 @@ func NewDatafile(path string, id int, readonly bool, maxKeySize uint32, maxValue
 		return nil, errors.Wrap(err, "error calling Stat()")
 	}
 
-	ra, err = mmap.Open(fn)
-	if err != nil {
-		return nil, err
+	if readonly {
+		ra, err = mmap.Open(fn)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	offset := stat.Size()
@@ -107,7 +109,9 @@ func (df *datafile) Name() string {
 
 func (df *datafile) Close() error {
 	defer func() {
-		df.ra.Close()
+		if df.ra != nil {
+			df.ra.Close()
+		}
 		df.r.Close()
 	}()
 
@@ -155,7 +159,10 @@ func (df *datafile) ReadAt(index, size int64) (e internal.Entry, err error) {
 
 	b := make([]byte, size)
 
-	if df.w == nil {
+	df.RLock()
+	defer df.RUnlock()
+
+	if df.ra != nil {
 		n, err = df.ra.ReadAt(b, index)
 	} else {
 		n, err = df.r.ReadAt(b, index)

internal/data/recover.go

@@ -6,9 +6,9 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/prologic/bitcask/internal"
-	"github.com/prologic/bitcask/internal/config"
-	"github.com/prologic/bitcask/internal/data/codec"
+	"git.mills.io/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal/config"
+	"git.mills.io/prologic/bitcask/internal/data/codec"
 )
 
 // CheckAndRecover checks and recovers the last datafile.
@@ -27,7 +27,7 @@ func CheckAndRecover(path string, cfg *config.Config) error {
 	f := dfs[len(dfs)-1]
 	recovered, err := recoverDatafile(f, cfg)
 	if err != nil {
-		return fmt.Errorf("recovering data file")
+		return fmt.Errorf("error recovering data file: %s", err)
 	}
 	if recovered {
 		if err := os.Remove(filepath.Join(path, "index")); err != nil {
@@ -48,8 +48,8 @@ func recoverDatafile(path string, cfg *config.Config) (recovered bool, err error
 			err = closeErr
 		}
 	}()
-	_, file := filepath.Split(path)
-	rPath := fmt.Sprintf("%s.recovered", file)
+	dir, file := filepath.Split(path)
+	rPath := filepath.Join(dir, fmt.Sprintf("%s.recovered", file))
 	fr, err := os.OpenFile(rPath, os.O_CREATE|os.O_WRONLY, os.ModePerm)
 	if err != nil {
 		return false, fmt.Errorf("creating the recovered datafile: %w", err)

internal/entry.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"hash/crc32"
+	"time"
 )
 
 // Entry represents a key/value in the database
@@ -10,15 +11,17 @@ type Entry struct {
 	Key      []byte
 	Offset   int64
 	Value    []byte
+	Expiry   *time.Time
 }
 
 // NewEntry creates a new `Entry` with the given `key` and `value`
-func NewEntry(key, value []byte) Entry {
+func NewEntry(key, value []byte, expiry *time.Time) Entry {
 	checksum := crc32.ChecksumIEEE(value)
 
 	return Entry{
 		Checksum: checksum,
 		Key:      key,
 		Value:    value,
+		Expiry:   expiry,
 	}
 }

internal/index/codec_index.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/pkg/errors"
 	art "github.com/plar/go-adaptive-radix-tree"
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 )
 
 var (
@@ -34,7 +34,7 @@ func readKeyBytes(r io.Reader, maxKeySize uint32) ([]byte, error) {
 		return nil, errors.Wrap(errTruncatedKeySize, err.Error())
 	}
 	size := binary.BigEndian.Uint32(s)
-	if size > uint32(maxKeySize) {
+	if maxKeySize > 0 && size > uint32(maxKeySize) {
 		return nil, errKeySizeTooLarge
 	}
 

internal/index/codec_index_test.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/pkg/errors"
 	art "github.com/plar/go-adaptive-radix-tree"
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 )
 
 const (

internal/index/index.go

@@ -4,7 +4,7 @@ import (
 	"os"
 
 	art "github.com/plar/go-adaptive-radix-tree"
-	"github.com/prologic/bitcask/internal"
+	"git.mills.io/prologic/bitcask/internal"
 )
 
 // Indexer is an interface for loading and saving the index (an Adaptive Radix Tree)

internal/index/ttl_index.go

@@ -0,0 +1,71 @@
+package index
+
+import (
+	"encoding/binary"
+	"io"
+	"os"
+	"time"
+
+	art "github.com/plar/go-adaptive-radix-tree"
+	"git.mills.io/prologic/bitcask/internal"
+)
+
+type ttlIndexer struct{}
+
+func NewTTLIndexer() Indexer {
+	return ttlIndexer{}
+}
+
+func (i ttlIndexer) Save(t art.Tree, path string) error {
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	buf := make([]byte, int64Size)
+	for it := t.Iterator(); it.HasNext(); {
+		node, err := it.Next()
+		if err != nil {
+			return err
+		}
+		// save key
+		err = writeBytes(node.Key(), f)
+		if err != nil {
+			return err
+		}
+		// save key ttl
+		binary.BigEndian.PutUint64(buf, uint64(node.Value().(time.Time).Unix()))
+		_, err = f.Write(buf)
+		if err != nil {
+			return err
+		}
+	}
+	return f.Sync()
+}
+
+func (i ttlIndexer) Load(path string, maxKeySize uint32) (art.Tree, bool, error) {
+	t := art.New()
+	if !internal.Exists(path) {
+		return t, false, nil
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return t, true, err
+	}
+	buf := make([]byte, int64Size)
+	for {
+		key, err := readKeyBytes(f, maxKeySize)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return t, true, err
+		}
+		_, err = io.ReadFull(f, buf)
+		if err != nil {
+			return t, true, err
+		}
+		expiry := time.Unix(int64(binary.BigEndian.Uint64(buf)), 0).UTC()
+		t.Insert(key, expiry)
+	}
+	return t, true, nil
+}

internal/index/ttl_index_test.go

@@ -0,0 +1,54 @@
+package index
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	art "github.com/plar/go-adaptive-radix-tree"
+	assert2 "github.com/stretchr/testify/assert"
+)
+
+func Test_TTLIndexer(t *testing.T) {
+	assert := assert2.New(t)
+	tempDir, err := ioutil.TempDir("", "bitcask")
+	assert.NoError(err)
+	defer os.RemoveAll(tempDir)
+
+	currTime := time.Date(2020, 12, 27, 0, 0, 0, 0, time.UTC)
+	trie := art.New()
+
+	t.Run("LoadEmpty", func(t *testing.T) {
+		newTrie, found, err := NewTTLIndexer().Load(filepath.Join(tempDir, "ttl_index"), 4)
+		assert.NoError(err)
+		assert.False(found)
+		assert.Equal(trie, newTrie)
+	})
+
+	t.Run("Save", func(t *testing.T) {
+		trie.Insert([]byte("key"), currTime)
+		err := NewTTLIndexer().Save(trie, filepath.Join(tempDir, "ttl_index"))
+		assert.NoError(err)
+		trie.Insert([]byte("foo"), currTime.Add(24*time.Hour))
+		err = NewTTLIndexer().Save(trie, filepath.Join(tempDir, "ttl_index"))
+		assert.NoError(err)
+		trie.Insert([]byte("key"), currTime.Add(-24*time.Hour))
+		err = NewTTLIndexer().Save(trie, filepath.Join(tempDir, "ttl_index"))
+		assert.NoError(err)
+	})
+
+	t.Run("Load", func(t *testing.T) {
+		newTrie, found, err := NewTTLIndexer().Load(filepath.Join(tempDir, "ttl_index"), 4)
+		assert.NoError(err)
+		assert.True(found)
+		assert.Equal(2, newTrie.Size())
+		value, found := newTrie.Search([]byte("key"))
+		assert.True(found)
+		assert.Equal(currTime.Add(-24*time.Hour), value)
+		value, found = newTrie.Search([]byte("foo"))
+		assert.True(found)
+		assert.Equal(currTime.Add(24*time.Hour), value)
+	})
+}

internal/metadata/metadata.go

@@ -0,0 +1,22 @@
+package metadata
+
+import (
+	"os"
+
+	"git.mills.io/prologic/bitcask/internal"
+)
+
+type MetaData struct {
+	IndexUpToDate    bool  `json:"index_up_to_date"`
+	ReclaimableSpace int64 `json:"reclaimable_space"`
+}
+
+func (m *MetaData) Save(path string, mode os.FileMode) error {
+	return internal.SaveJsonToFile(m, path, mode)
+}
+
+func Load(path string) (*MetaData, error) {
+	var m MetaData
+	err := internal.LoadFromJsonFile(path, &m)
+	return &m, err
+}

internal/mocks/datafile.go

@@ -2,7 +2,7 @@
 
 package mocks
 
-import internal "github.com/prologic/bitcask/internal"
+import internal "git.mills.io/prologic/bitcask/internal"
 import mock "github.com/stretchr/testify/mock"
 
 // Datafile is an autogenerated mock type for the Datafile type

internal/utils.go

@@ -1,7 +1,9 @@
 package internal
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
@@ -63,3 +65,48 @@ func ParseIds(fns []string) ([]int, error) {
 	sort.Ints(ids)
 	return ids, nil
 }
+
+// Copy copies source contents to destination
+func Copy(src, dst string, exclude []string) error {
+	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+		relPath := strings.Replace(path, src, "", 1)
+		if relPath == "" {
+			return nil
+		}
+		for _, e := range exclude {
+			matched, err := filepath.Match(e, info.Name())
+			if err != nil {
+				return err
+			}
+			if matched {
+				return nil
+			}
+		}
+		if info.IsDir() {
+			return os.Mkdir(filepath.Join(dst, relPath), info.Mode())
+		}
+		var data, err1 = ioutil.ReadFile(filepath.Join(src, relPath))
+		if err1 != nil {
+			return err1
+		}
+		return ioutil.WriteFile(filepath.Join(dst, relPath), data, info.Mode())
+	})
+}
+
+// SaveJsonToFile converts v into json and store in file identified by path
+func SaveJsonToFile(v interface{}, path string, mode os.FileMode) error {
+	b, err := json.Marshal(v)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(path, b, mode)
+}
+
+// LoadFromJsonFile reads file located at `path` and put its content in json format in v
+func LoadFromJsonFile(path string, v interface{}) error {
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(b, v)
+}

internal/utils_test.go

@@ -0,0 +1,108 @@
+package internal
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_Copy(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("CopyDir", func(t *testing.T) {
+		tempsrc, err := ioutil.TempDir("", "test")
+		assert.NoError(err)
+		defer os.RemoveAll(tempsrc)
+		var f *os.File
+
+		tempdir, err := ioutil.TempDir(tempsrc, "")
+		assert.NoError(err)
+
+		f, err = os.OpenFile(filepath.Join(tempsrc, "file1"), os.O_WRONLY|os.O_CREATE, 0755)
+		assert.NoError(err)
+		n, err := f.WriteString("test123")
+		assert.Equal(7, n)
+		assert.NoError(err)
+		f.Close()
+
+		f, err = os.OpenFile(filepath.Join(tempsrc, "file2"), os.O_WRONLY|os.O_CREATE, 0755)
+		assert.NoError(err)
+		n, err = f.WriteString("test1234")
+		assert.Equal(8, n)
+		assert.NoError(err)
+		f.Close()
+
+		f, err = os.OpenFile(filepath.Join(tempsrc, "file3"), os.O_WRONLY|os.O_CREATE, 0755)
+		assert.NoError(err)
+		f.Close()
+
+		tempdst, err := ioutil.TempDir("", "backup")
+		assert.NoError(err)
+		defer os.RemoveAll(tempdst)
+		err = Copy(tempsrc, tempdst, []string{"file3"})
+		assert.NoError(err)
+		buf := make([]byte, 10)
+
+		exists := Exists(filepath.Join(tempdst, filepath.Base(tempdir)))
+		assert.Equal(true, exists)
+
+		f, err = os.Open(filepath.Join(tempdst, "file1"))
+		assert.NoError(err)
+		n, err = f.Read(buf[:7])
+		assert.NoError(err)
+		assert.Equal(7, n)
+		assert.Equal([]byte("test123"), buf[:7])
+		_, err = f.Read(buf)
+		assert.Equal(io.EOF, err)
+		f.Close()
+
+		f, err = os.Open(filepath.Join(tempdst, "file2"))
+		assert.NoError(err)
+		n, err = f.Read(buf[:8])
+		assert.NoError(err)
+		assert.Equal(8, n)
+		assert.Equal([]byte("test1234"), buf[:8])
+		_, err = f.Read(buf)
+		assert.Equal(io.EOF, err)
+		f.Close()
+
+		exists = Exists(filepath.Join(tempdst, "file3"))
+		assert.Equal(false, exists)
+	})
+}
+
+func Test_SaveAndLoad(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("save and load", func(t *testing.T) {
+		tempdir, err := ioutil.TempDir("", "bitcask")
+		assert.NoError(err)
+		defer os.RemoveAll(tempdir)
+		type test struct {
+			Value bool `json:"value"`
+		}
+		m := test{Value: true}
+		err = SaveJsonToFile(&m, filepath.Join(tempdir, "meta.json"), 0755)
+		assert.NoError(err)
+		m1 := test{}
+		err = LoadFromJsonFile(filepath.Join(tempdir, "meta.json"), &m1)
+		assert.NoError(err)
+		assert.Equal(m, m1)
+	})
+
+	t.Run("save and load error", func(t *testing.T) {
+		tempdir, err := ioutil.TempDir("", "bitcask")
+		assert.NoError(err)
+		defer os.RemoveAll(tempdir)
+		type test struct {
+			Value bool `json:"value"`
+		}
+		err = SaveJsonToFile(make(chan int), filepath.Join(tempdir, "meta.json"), 0755)
+		assert.Error(err)
+		m1 := test{}
+		err = LoadFromJsonFile(filepath.Join(tempdir, "meta.json"), &m1)
+		assert.Error(err)
+	})
+}

options.go

@@ -3,7 +3,7 @@ package bitcask
 import (
 	"os"
 
-	"github.com/prologic/bitcask/internal/config"
+	"git.mills.io/prologic/bitcask/internal/config"
 )
 
 const (
@@ -26,11 +26,26 @@ const (
 	DefaultSync = false
 
 	// DefaultAutoRecovery is the default auto-recovery action.
+
+	CurrentDBVersion = uint32(1)
 )
 
 // Option is a function that takes a config struct and modifies it
 type Option func(*config.Config) error
 
+func withConfig(src *config.Config) Option {
+	return func(cfg *config.Config) error {
+		cfg.MaxDatafileSize = src.MaxDatafileSize
+		cfg.MaxKeySize = src.MaxKeySize
+		cfg.MaxValueSize = src.MaxValueSize
+		cfg.Sync = src.Sync
+		cfg.AutoRecovery = src.AutoRecovery
+		cfg.DirFileModeBeforeUmask = src.DirFileModeBeforeUmask
+		cfg.FileFileModeBeforeUmask = src.FileFileModeBeforeUmask
+		return nil
+	}
+}
+
 // WithAutoRecovery sets auto recovery of data and index file recreation.
 // IMPORTANT: This flag MUST BE used only if a proper backup was made of all
 // the existing datafiles.
@@ -98,5 +113,6 @@ func newDefaultConfig() *config.Config {
 		Sync:                    DefaultSync,
 		DirFileModeBeforeUmask:  DefaultDirFileModeBeforeUmask,
 		FileFileModeBeforeUmask: DefaultFileFileModeBeforeUmask,
+		DBVersion:               CurrentDBVersion,
 	}
 }

scripts/migrations/v0_to_v1.go

@@ -0,0 +1,159 @@
+package migrations
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+
+	"git.mills.io/prologic/bitcask/internal"
+)
+
+const (
+	keySize                 = 4
+	valueSize               = 8
+	checksumSize            = 4
+	ttlSize                 = 8
+	defaultDatafileFilename = "%09d.data"
+)
+
+func ApplyV0ToV1(dir string, maxDatafileSize int) error {
+	temp, err := prepare(dir)
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(temp)
+	err = apply(dir, temp, maxDatafileSize)
+	if err != nil {
+		return err
+	}
+	return cleanup(dir, temp)
+}
+
+func prepare(dir string) (string, error) {
+	return ioutil.TempDir(dir, "migration")
+}
+
+func apply(dir, temp string, maxDatafileSize int) error {
+	datafilesPath, err := internal.GetDatafiles(dir)
+	if err != nil {
+		return err
+	}
+	var id, newOffset int
+	datafile, err := getNewDatafile(temp, id)
+	if err != nil {
+		return err
+	}
+	id++
+	for _, p := range datafilesPath {
+		df, err := os.Open(p)
+		if err != nil {
+			return err
+		}
+		var off int64
+		for {
+			entry, err := getSingleEntry(df, off)
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				return err
+			}
+			if newOffset+len(entry) > maxDatafileSize {
+				err = datafile.Sync()
+				if err != nil {
+					return err
+				}
+				datafile, err = getNewDatafile(temp, id)
+				if err != nil {
+					return err
+				}
+				id++
+				newOffset = 0
+			}
+			newEntry := make([]byte, len(entry)+ttlSize)
+			copy(newEntry[:len(entry)], entry)
+			n, err := datafile.Write(newEntry)
+			if err != nil {
+				return err
+			}
+			newOffset += n
+			off += int64(len(entry))
+		}
+	}
+	return datafile.Sync()
+}
+
+func cleanup(dir, temp string) error {
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return err
+	}
+	for _, file := range files {
+		if !file.IsDir() {
+			err := os.RemoveAll(path.Join([]string{dir, file.Name()}...))
+			if err != nil {
+				return err
+			}
+		}
+	}
+	files, err = ioutil.ReadDir(temp)
+	if err != nil {
+		return err
+	}
+	for _, file := range files {
+		err := os.Rename(
+			path.Join([]string{temp, file.Name()}...),
+			path.Join([]string{dir, file.Name()}...),
+		)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func getNewDatafile(path string, id int) (*os.File, error) {
+	fn := filepath.Join(path, fmt.Sprintf(defaultDatafileFilename, id))
+	return os.OpenFile(fn, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0640)
+}
+
+func getSingleEntry(f *os.File, offset int64) ([]byte, error) {
+	prefixBuf, err := readPrefix(f, offset)
+	if err != nil {
+		return nil, err
+	}
+	actualKeySize, actualValueSize := getKeyValueSize(prefixBuf)
+	entryBuf, err := read(f, uint64(actualKeySize)+actualValueSize+checksumSize, offset+keySize+valueSize)
+	if err != nil {
+		return nil, err
+	}
+	return append(prefixBuf, entryBuf...), nil
+}
+
+func readPrefix(f *os.File, offset int64) ([]byte, error) {
+	prefixBuf := make([]byte, keySize+valueSize)
+	_, err := f.ReadAt(prefixBuf, offset)
+	if err != nil {
+		return nil, err
+	}
+	return prefixBuf, nil
+}
+
+func read(f *os.File, bufSize uint64, offset int64) ([]byte, error) {
+	buf := make([]byte, bufSize)
+	_, err := f.ReadAt(buf, offset)
+	if err != nil {
+		return nil, err
+	}
+	return buf, nil
+}
+
+func getKeyValueSize(buf []byte) (uint32, uint64) {
+	actualKeySize := binary.BigEndian.Uint32(buf[:keySize])
+	actualValueSize := binary.BigEndian.Uint64(buf[keySize:])
+	return actualKeySize, actualValueSize
+}

scripts/migrations/v0_to_v1_test.go

@@ -0,0 +1,58 @@
+package migrations
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_ApplyV0ToV1(t *testing.T) {
+	assert := assert.New(t)
+	testdir, err := ioutil.TempDir("/tmp", "bitcask")
+	assert.NoError(err)
+	defer os.RemoveAll(testdir)
+	w0, err := os.OpenFile(filepath.Join(testdir, "000000000.data"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
+	assert.NoError(err)
+	w1, err := os.OpenFile(filepath.Join(testdir, "000000001.data"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
+	assert.NoError(err)
+	w2, err := os.OpenFile(filepath.Join(testdir, "000000002.data"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
+	assert.NoError(err)
+	defer w0.Close()
+	defer w1.Close()
+	defer w2.Close()
+	buf := make([]byte, 104)
+	binary.BigEndian.PutUint32(buf[:4], 5)
+	binary.BigEndian.PutUint64(buf[4:12], 7)
+	copy(buf[12:28], "mykeymyvalue0AAA")
+	binary.BigEndian.PutUint32(buf[28:32], 3)
+	binary.BigEndian.PutUint64(buf[32:40], 5)
+	copy(buf[40:52], "keyvalue0BBB")
+	_, err = w0.Write(buf[:52])
+	assert.NoError(err)
+	_, err = w1.Write(buf[:52])
+	assert.NoError(err)
+	_, err = w2.Write(buf[:52])
+	assert.NoError(err)
+	err = ApplyV0ToV1(testdir, 104)
+	assert.NoError(err)
+	r0, err := os.Open(filepath.Join(testdir, "000000000.data"))
+	assert.NoError(err)
+	defer r0.Close()
+	n, err := io.ReadFull(r0, buf)
+	assert.NoError(err)
+	assert.Equal(104, n)
+	assert.Equal("0000000500000000000000076d796b65796d7976616c75653041414100000000000000000000000300000000000000056b657976616c75653042424200000000000000000000000500000000000000076d796b65796d7976616c7565304141410000000000000000", hex.EncodeToString(buf))
+	r1, err := os.Open(filepath.Join(testdir, "000000001.data"))
+	assert.NoError(err)
+	defer r1.Close()
+	n, err = io.ReadFull(r1, buf[:100])
+	assert.NoError(err)
+	assert.Equal(100, n)
+	assert.Equal("0000000300000000000000056b657976616c75653042424200000000000000000000000500000000000000076d796b65796d7976616c75653041414100000000000000000000000300000000000000056b657976616c7565304242420000000000000000", hex.EncodeToString(buf[:100]))
+}

tools/release.sh

@@ -26,7 +26,7 @@ echo "Releasing ${TAG} ..."
 git-chglog --next-tag="${TAG}" --output CHANGELOG.md
 git commit -a -m "Update CHANGELOG for ${TAG}"
 git tag -a -s -m "Release ${TAG}" "${TAG}"
-git push --tags
+git push && git push --tags
 goreleaser release \
   --rm-dist \
   --release-notes <(git-chglog "${TAG}" | tail -n+5)