mirror of
https://github.com/gogrlx/bitcask.git
synced 2026-04-02 11:09:01 -07:00
Compare commits
24 Commits
v0.3.13
...
dependabot
| Author | SHA1 | Date | |
|---|---|---|---|
|
edef0ce549 | ||
|
4ec5b07eea | ||
|
|
5ed43e5a20 | ||
| edd32cad0a | |||
| d23c355e72 | |||
| 40425394d7 | |||
| f4cc0fb434 | |||
| 7d4174d5b1 | |||
|
5429693cc8 | ||
|
2c57c950f8 | ||
|
21a824e13e | ||
| 2279245b8c | |||
| 91d4db63d5 | |||
| 849192f709 | |||
| a4fc2cf4e8 | |||
| 609de833eb | |||
|
|
9b0daa8a30 | ||
| ef187f8315 | |||
|
|
b094cd33d3 | ||
| 3ff8937205 | |||
| 2ccca759ce | |||
| 92535e654b | |||
| c4a7ad7a7f | |||
| e64646fa8f |
17
.drone.yml
17
.drone.yml
@@ -5,12 +5,6 @@ name: default
|
||||
steps:
|
||||
- name: build & run tests
|
||||
image: r.mills.io/prologic/golang-alpine
|
||||
volumes:
|
||||
- name: gomodcache
|
||||
path: /go/pkg/mod/cache
|
||||
privileged: true
|
||||
environment:
|
||||
GOPROXY: https://goproxy.mills.io
|
||||
commands:
|
||||
- make build
|
||||
- make test
|
||||
@@ -25,7 +19,10 @@ steps:
|
||||
- success
|
||||
- failure
|
||||
|
||||
volumes:
|
||||
- name: gomodcache
|
||||
host:
|
||||
path: /var/lib/cache/go
|
||||
trigger:
|
||||
branch:
|
||||
- master
|
||||
event:
|
||||
- tag
|
||||
- push
|
||||
- pull_request
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -5,6 +5,7 @@
|
||||
|
||||
/tmp
|
||||
/dist
|
||||
/cacheDb
|
||||
/coverage.txt
|
||||
|
||||
/bitcask
|
||||
|
||||
2
AUTHORS
2
AUTHORS
@@ -17,3 +17,5 @@ Yash Chandra <yashschandra@gmail.com>
|
||||
Yury Fedorov orlangure
|
||||
o2gy84 <o2gy84@gmail.com>
|
||||
garsue <labs.garsue@gmail.com>
|
||||
biozz <ielfimov@gmail.com>
|
||||
jason3gb <jason3gb@gmail.com>
|
||||
64
CHANGELOG.md
64
CHANGELOG.md
@@ -1,4 +1,64 @@
|
||||
|
||||
<a name="v1.0.2"></a>
|
||||
## [v1.0.2](https://git.mills.io/prologic/bitcask/compare/v1.0.1...v1.0.2) (2021-11-01)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Fix a data race in Datafile.ReadAt()
|
||||
* Fix release tool
|
||||
|
||||
|
||||
<a name="v1.0.1"></a>
|
||||
## [v1.0.1](https://git.mills.io/prologic/bitcask/compare/v1.0.0...v1.0.1) (2021-10-31)
|
||||
|
||||
### Features
|
||||
|
||||
* Add ErrBadConfig and ErrBadMetadata as errors that consumers can check and use (#241)
|
||||
* Add key prefix matching to KEYS command (#237)
|
||||
|
||||
### Updates
|
||||
|
||||
* Update CHANGELOG for v1.0.1
|
||||
* Update image target
|
||||
|
||||
|
||||
<a name="v1.0.0"></a>
|
||||
## [v1.0.0](https://git.mills.io/prologic/bitcask/compare/1.0.0...v1.0.0) (2021-07-24)
|
||||
|
||||
### Updates
|
||||
|
||||
* Update CHANGELOG for v1.0.0
|
||||
|
||||
|
||||
<a name="1.0.0"></a>
|
||||
## [1.0.0](https://git.mills.io/prologic/bitcask/compare/v0.3.14...1.0.0) (2021-07-24)
|
||||
|
||||
### Updates
|
||||
|
||||
* Update CHANGELOG for 1.0.0
|
||||
* Update README
|
||||
|
||||
|
||||
<a name="v0.3.14"></a>
|
||||
## [v0.3.14](https://git.mills.io/prologic/bitcask/compare/v0.3.13...v0.3.14) (2021-07-21)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Fix runGC behaviour to correctly delete all expired keys (#229)
|
||||
* Fix missing push event
|
||||
* Fix how CI is triggered
|
||||
* Fix README Go Reference badge
|
||||
* Fix README badges
|
||||
|
||||
### Features
|
||||
|
||||
* Add RangeScan() support (#160)
|
||||
|
||||
### Updates
|
||||
|
||||
* Update CHANGELOG for v0.3.14
|
||||
|
||||
|
||||
<a name="v0.3.13"></a>
|
||||
## [v0.3.13](https://git.mills.io/prologic/bitcask/compare/v0.3.12...v0.3.13) (2021-07-16)
|
||||
|
||||
@@ -10,6 +70,10 @@
|
||||
|
||||
* Add Drone CI config
|
||||
|
||||
### Updates
|
||||
|
||||
* Update CHANGELOG for v0.3.13
|
||||
|
||||
|
||||
<a name="v0.3.12"></a>
|
||||
## [v0.3.12](https://git.mills.io/prologic/bitcask/compare/v0.3.11...v0.3.12) (2021-07-13)
|
||||
|
||||
34
Makefile
34
Makefile
@@ -3,6 +3,10 @@
|
||||
CGO_ENABLED=0
|
||||
VERSION=$(shell git describe --abbrev=0 --tags)
|
||||
COMMIT=$(shell git rev-parse --short HEAD)
|
||||
BUILD=$(shell git show -s --pretty=format:%cI)
|
||||
GOCMD=go
|
||||
|
||||
DESTDIR=/usr/local/bin
|
||||
|
||||
all: dev
|
||||
|
||||
@@ -11,46 +15,52 @@ dev: build
|
||||
@./bitcaskd --version
|
||||
|
||||
build: clean generate
|
||||
@go build \
|
||||
@$(GOCMD) build \
|
||||
-tags "netgo static_build" -installsuffix netgo \
|
||||
-ldflags "-w -X $(shell go list)/internal.Version=$(VERSION) -X $(shell go list)/internal.Commit=$(COMMIT)" \
|
||||
-ldflags "-w -X $(shell go list)/internal.Version=$(VERSION) -X $(shell go list)/internal.Commit=$(COMMIT) -X $(shell go list)/internal.Build=$(BUILD)" \
|
||||
./cmd/bitcask/...
|
||||
@go build \
|
||||
@$(GOCMD) build \
|
||||
-tags "netgo static_build" -installsuffix netgo \
|
||||
-ldflags "-w -X $(shell go list)/internal.Version=$(VERSION) -X $(shell go list)/internal.Commit=$(COMMIT)" \
|
||||
-ldflags "-w -X $(shell go list)/internal.Version=$(VERSION) -X $(shell go list)/internal.Commit=$(COMMIT) -X $(shell go list)/internal.Build=$(BUILD)" \
|
||||
./cmd/bitcaskd/...
|
||||
|
||||
generate:
|
||||
@go generate $(shell go list)/...
|
||||
@$(GOCMD) generate $(shell go list)/...
|
||||
|
||||
install: build
|
||||
@go install ./cmd/bitcask/...
|
||||
@go install ./cmd/bitcaskd/...
|
||||
@install -D -m 755 bitcask $(DESTDIR)/bitcask
|
||||
@install -D -m 755 bitcaskd $(DESTDIR)/bitcaskd
|
||||
|
||||
ifeq ($(PUBLISH), 1)
|
||||
image:
|
||||
@docker build -t prologic/bitcask .
|
||||
@docker build --build-arg VERSION="$(VERSION)" --build-arg COMMIT="$(COMMIT)" -t prologic/bitcask .
|
||||
@docker push prologic/bitcask
|
||||
else
|
||||
image:
|
||||
@docker build --build-arg VERSION="$(VERSION)" --build-arg COMMIT="$(COMMIT)" -t prologic/bitcask .
|
||||
endif
|
||||
|
||||
release:
|
||||
@./tools/release.sh
|
||||
|
||||
profile: build
|
||||
@go test -cpuprofile cpu.prof -memprofile mem.prof -v -bench .
|
||||
@$(GOCMD) test -cpuprofile cpu.prof -memprofile mem.prof -v -bench .
|
||||
|
||||
bench: build
|
||||
@go test -v -run=XXX -benchmem -bench=. .
|
||||
@$(GOCMD) test -v -run=XXX -benchmem -bench=. .
|
||||
|
||||
mocks:
|
||||
@mockery -all -case underscore -output ./internal/mocks -recursive
|
||||
|
||||
test: build
|
||||
@go test -v \
|
||||
@$(GOCMD) test -v \
|
||||
-cover -coverprofile=coverage.txt -covermode=atomic \
|
||||
-coverpkg=$(shell go list) \
|
||||
-race \
|
||||
.
|
||||
|
||||
setup:
|
||||
@go get github.com/vektra/mockery/...
|
||||
@$(GOCMD) get github.com/vektra/mockery/...
|
||||
|
||||
clean:
|
||||
@git clean -f -d -X
|
||||
|
||||
35
README.md
35
README.md
@@ -1,17 +1,8 @@
|
||||
# bitcask
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
[](https://codecov.io/gh/prologic/bitcask)
|
||||
[](https://goreportcard.com/report/prologic/bitcask)
|
||||
[](https://codebeat.co/projects/github-com-prologic-bitcask-master)
|
||||
[](https://godoc.org/git.mills.io/prologic/bitcask)
|
||||
[](https://git.mills.io/prologic/bitcask)
|
||||
[](https://sourcegraph.com/git.mills.io/prologic/bitcask?badge)
|
||||
[](https://www.tickgit.com/browse?repo=git.mills.io/prologic/bitcask)
|
||||
[](https://ci.mills.io/prologic/bitcask)
|
||||
[](https://goreportcard.com/report/git.mills.io/prologic/bitcask)
|
||||
[](https://pkg.go.dev/git.mills.io/prologic/bitcask)
|
||||
|
||||
A high performance Key/Value store written in [Go](https://golang.org) with a predictable read/write performance and high throughput. Uses a [Bitcask](https://en.wikipedia.org/wiki/Bitcask) on-disk layout (LSM+WAL) similar to [Riak](https://riak.com/)
|
||||
|
||||
@@ -229,24 +220,14 @@ Support the ongoing development of Bitcask!
|
||||
|
||||
- Become a [Sponsor](https://www.patreon.com/prologic)
|
||||
|
||||
## Stargazers over time
|
||||
|
||||
[](https://starcharts.herokuapp.com/prologic/bitcask)
|
||||
|
||||
## Contributors
|
||||
|
||||
Thank you to all those that have contributed to this project, battle-tested it, used it in their own projects or products, fixed bugs, improved performance and even fix tiny typos in documentation! Thank you and keep contributing!
|
||||
Thank you to all those that have contributed to this project, battle-tested it,
|
||||
used it in their own projects or products, fixed bugs, improved performance
|
||||
and even fix tiny typos in documentation! Thank you and keep contributing!
|
||||
|
||||
You can find an [AUTHORS](/AUTHORS) file where we keep a list of contributors to the project. If you contribute a PR please consider adding your name there. There is also GitHub's own [Contributors](https://git.mills.io/prologic/bitcask/graphs/contributors) statistics.
|
||||
|
||||
[](https://sourcerer.io/fame/prologic/prologic/bitcask/links/0)
|
||||
[](https://sourcerer.io/fame/prologic/prologic/bitcask/links/1)
|
||||
[](https://sourcerer.io/fame/prologic/prologic/bitcask/links/2)
|
||||
[](https://sourcerer.io/fame/prologic/prologic/bitcask/links/3)
|
||||
[](https://sourcerer.io/fame/prologic/prologic/bitcask/links/4)
|
||||
[](https://sourcerer.io/fame/prologic/prologic/bitcask/links/5)
|
||||
[](https://sourcerer.io/fame/prologic/prologic/bitcask/links/6)
|
||||
[](https://sourcerer.io/fame/prologic/prologic/bitcask/links/7)
|
||||
You can find an [AUTHORS](/AUTHORS) file where we keep a list of contributors
|
||||
to the project. If you contribute a PR please consider adding your name there.
|
||||
|
||||
## Related Projects
|
||||
|
||||
|
||||
228
bitcask.go
228
bitcask.go
@@ -1,7 +1,7 @@
|
||||
package bitcask
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/abcum/lcp"
|
||||
"github.com/gofrs/flock"
|
||||
art "github.com/plar/go-adaptive-radix-tree"
|
||||
log "github.com/sirupsen/logrus"
|
||||
@@ -31,40 +32,6 @@ const (
|
||||
ttlIndexFile = "ttl_index"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrKeyNotFound is the error returned when a key is not found
|
||||
ErrKeyNotFound = errors.New("error: key not found")
|
||||
|
||||
// ErrKeyTooLarge is the error returned for a key that exceeds the
|
||||
// maximum allowed key size (configured with WithMaxKeySize).
|
||||
ErrKeyTooLarge = errors.New("error: key too large")
|
||||
|
||||
// ErrKeyExpired is the error returned when a key is queried which has
|
||||
// already expired (due to ttl)
|
||||
ErrKeyExpired = errors.New("error: key expired")
|
||||
|
||||
// ErrEmptyKey is the error returned for a value with an empty key.
|
||||
ErrEmptyKey = errors.New("error: empty key")
|
||||
|
||||
// ErrValueTooLarge is the error returned for a value that exceeds the
|
||||
// maximum allowed value size (configured with WithMaxValueSize).
|
||||
ErrValueTooLarge = errors.New("error: value too large")
|
||||
|
||||
// ErrChecksumFailed is the error returned if a key/value retrieved does
|
||||
// not match its CRC checksum
|
||||
ErrChecksumFailed = errors.New("error: checksum failed")
|
||||
|
||||
// ErrDatabaseLocked is the error returned if the database is locked
|
||||
// (typically opened by another process)
|
||||
ErrDatabaseLocked = errors.New("error: database locked")
|
||||
|
||||
ErrInvalidVersion = errors.New("error: invalid db version")
|
||||
|
||||
// ErrMergeInProgress is the error returned if merge is called when already a merge
|
||||
// is in progress
|
||||
ErrMergeInProgress = errors.New("error: merge already in progress")
|
||||
)
|
||||
|
||||
// Bitcask is a struct that represents a on-disk LSM and WAL data structure
|
||||
// and in-memory hash of key/value pairs as per the Bitcask paper and seen
|
||||
// in the Riak database.
|
||||
@@ -274,6 +241,41 @@ func (b *Bitcask) delete(key []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sift iterates over all keys in the database calling the function `f` for
|
||||
// each key. If the KV pair is expired or the function returns true, that key is
|
||||
// deleted from the database.
|
||||
// If the function returns an error on any key, no further keys are processed, no
|
||||
// keys are deleted, and the first error is returned.
|
||||
func (b *Bitcask) Sift(f func(key []byte) (bool, error)) (err error) {
|
||||
keysToDelete := art.New()
|
||||
|
||||
b.mu.RLock()
|
||||
b.trie.ForEach(func(node art.Node) bool {
|
||||
if b.isExpired(node.Key()) {
|
||||
keysToDelete.Insert(node.Key(), true)
|
||||
return true
|
||||
}
|
||||
var shouldDelete bool
|
||||
if shouldDelete, err = f(node.Key()); err != nil {
|
||||
return false
|
||||
} else if shouldDelete {
|
||||
keysToDelete.Insert(node.Key(), true)
|
||||
}
|
||||
return true
|
||||
})
|
||||
b.mu.RUnlock()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
keysToDelete.ForEach(func(node art.Node) (cont bool) {
|
||||
b.delete(node.Key())
|
||||
return true
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// DeleteAll deletes all the keys. If an I/O error occurs the error is returned.
|
||||
func (b *Bitcask) DeleteAll() (err error) {
|
||||
b.mu.RLock()
|
||||
@@ -296,8 +298,11 @@ func (b *Bitcask) DeleteAll() (err error) {
|
||||
|
||||
// Scan performs a prefix scan of keys matching the given prefix and calling
|
||||
// the function `f` with the keys found. If the function returns an error
|
||||
// no further keys are processed and the first error returned.
|
||||
// no further keys are processed and the first error is returned.
|
||||
func (b *Bitcask) Scan(prefix []byte, f func(key []byte) error) (err error) {
|
||||
b.mu.RLock()
|
||||
defer b.mu.RUnlock()
|
||||
|
||||
b.trie.ForEachPrefix(prefix, func(node art.Node) bool {
|
||||
// Skip the root node
|
||||
if len(node.Key()) == 0 {
|
||||
@@ -312,6 +317,132 @@ func (b *Bitcask) Scan(prefix []byte, f func(key []byte) error) (err error) {
|
||||
return
|
||||
}
|
||||
|
||||
// SiftScan iterates over all keys in the database beginning with the given
|
||||
// prefix, calling the function `f` for each key. If the KV pair is expired or
|
||||
// the function returns true, that key is deleted from the database.
|
||||
// If the function returns an error on any key, no further keys are processed,
|
||||
// no keys are deleted, and the first error is returned.
|
||||
func (b *Bitcask) SiftScan(prefix []byte, f func(key []byte) (bool, error)) (err error) {
|
||||
keysToDelete := art.New()
|
||||
|
||||
b.mu.RLock()
|
||||
b.trie.ForEachPrefix(prefix, func(node art.Node) bool {
|
||||
// Skip the root node
|
||||
if len(node.Key()) == 0 {
|
||||
return true
|
||||
}
|
||||
if b.isExpired(node.Key()) {
|
||||
keysToDelete.Insert(node.Key(), true)
|
||||
return true
|
||||
}
|
||||
var shouldDelete bool
|
||||
if shouldDelete, err = f(node.Key()); err != nil {
|
||||
return false
|
||||
} else if shouldDelete {
|
||||
keysToDelete.Insert(node.Key(), true)
|
||||
}
|
||||
return true
|
||||
})
|
||||
b.mu.RUnlock()
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
keysToDelete.ForEach(func(node art.Node) (cont bool) {
|
||||
b.delete(node.Key())
|
||||
return true
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Range performs a range scan of keys matching a range of keys between the
|
||||
// start key and end key and calling the function `f` with the keys found.
|
||||
// If the function returns an error no further keys are processed and the
|
||||
// first error returned.
|
||||
func (b *Bitcask) Range(start, end []byte, f func(key []byte) error) (err error) {
|
||||
if bytes.Compare(start, end) == 1 {
|
||||
return ErrInvalidRange
|
||||
}
|
||||
|
||||
commonPrefix := lcp.LCP(start, end)
|
||||
if commonPrefix == nil {
|
||||
return ErrInvalidRange
|
||||
}
|
||||
|
||||
b.mu.RLock()
|
||||
defer b.mu.RUnlock()
|
||||
|
||||
b.trie.ForEachPrefix(commonPrefix, func(node art.Node) bool {
|
||||
if bytes.Compare(node.Key(), start) >= 0 && bytes.Compare(node.Key(), end) <= 0 {
|
||||
if err = f(node.Key()); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
} else if bytes.Compare(node.Key(), start) >= 0 && bytes.Compare(node.Key(), end) > 0 {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// SiftRange performs a range scan of keys matching a range of keys between the
|
||||
// start key and end key and calling the function `f` with the keys found.
|
||||
// If the KV pair is expired or the function returns true, that key is deleted
|
||||
// from the database.
|
||||
// If the function returns an error on any key, no further keys are processed, no
|
||||
// keys are deleted, and the first error is returned.
|
||||
func (b *Bitcask) SiftRange(start, end []byte, f func(key []byte) (bool, error)) (err error) {
|
||||
if bytes.Compare(start, end) == 1 {
|
||||
return ErrInvalidRange
|
||||
}
|
||||
|
||||
commonPrefix := lcp.LCP(start, end)
|
||||
if commonPrefix == nil {
|
||||
return ErrInvalidRange
|
||||
}
|
||||
|
||||
keysToDelete := art.New()
|
||||
|
||||
b.mu.RLock()
|
||||
b.trie.ForEachPrefix(commonPrefix, func(node art.Node) bool {
|
||||
if bytes.Compare(node.Key(), start) >= 0 && bytes.Compare(node.Key(), end) <= 0 {
|
||||
if b.isExpired(node.Key()) {
|
||||
keysToDelete.Insert(node.Key(), true)
|
||||
return true
|
||||
}
|
||||
var shouldDelete bool
|
||||
if shouldDelete, err = f(node.Key()); err != nil {
|
||||
return false
|
||||
} else if shouldDelete {
|
||||
keysToDelete.Insert(node.Key(), true)
|
||||
}
|
||||
return true
|
||||
} else if bytes.Compare(node.Key(), start) >= 0 && bytes.Compare(node.Key(), end) > 0 {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
b.mu.RUnlock()
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
|
||||
keysToDelete.ForEach(func(node art.Node) (cont bool) {
|
||||
b.delete(node.Key())
|
||||
return true
|
||||
})
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Len returns the total number of keys in the database
|
||||
func (b *Bitcask) Len() int {
|
||||
b.mu.RLock()
|
||||
@@ -349,21 +480,29 @@ func (b *Bitcask) RunGC() error {
|
||||
// runGC deletes all keys that are expired
|
||||
// caller function should take care of the locking when calling this method
|
||||
func (b *Bitcask) runGC() (err error) {
|
||||
keysToDelete := art.New()
|
||||
|
||||
b.ttlIndex.ForEach(func(node art.Node) (cont bool) {
|
||||
if !b.isExpired(node.Key()) {
|
||||
// later, return false here when the ttlIndex is sorted
|
||||
return true
|
||||
}
|
||||
if err = b.delete(node.Key()); err != nil {
|
||||
return false
|
||||
}
|
||||
keysToDelete.Insert(node.Key(), true)
|
||||
//keysToDelete = append(keysToDelete, node.Key())
|
||||
return true
|
||||
})
|
||||
return
|
||||
|
||||
keysToDelete.ForEach(func(node art.Node) (cont bool) {
|
||||
b.delete(node.Key())
|
||||
return true
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Fold iterates over all keys in the database calling the function `f` for
|
||||
// each key. If the function returns an error, no further keys are processed
|
||||
// and the error returned.
|
||||
// and the error is returned.
|
||||
func (b *Bitcask) Fold(f func(key []byte) error) (err error) {
|
||||
b.mu.RLock()
|
||||
defer b.mu.RUnlock()
|
||||
@@ -386,8 +525,7 @@ func (b *Bitcask) get(key []byte) (internal.Entry, error) {
|
||||
if !found {
|
||||
return internal.Entry{}, ErrKeyNotFound
|
||||
}
|
||||
if expired := b.isExpired(key); expired {
|
||||
_ = b.delete(key) // we don't care if it doesnt succeed
|
||||
if b.isExpired(key) {
|
||||
return internal.Entry{}, ErrKeyExpired
|
||||
}
|
||||
|
||||
@@ -700,7 +838,7 @@ func Open(path string, options ...Option) (*Bitcask, error) {
|
||||
if internal.Exists(configPath) {
|
||||
cfg, err = config.Load(configPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, &ErrBadConfig{err}
|
||||
}
|
||||
} else {
|
||||
cfg = newDefaultConfig()
|
||||
@@ -722,7 +860,7 @@ func Open(path string, options ...Option) (*Bitcask, error) {
|
||||
|
||||
meta, err = loadMetadata(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, &ErrBadMetadata{err}
|
||||
}
|
||||
|
||||
bitcask := &Bitcask{
|
||||
|
||||
354
bitcask_test.go
354
bitcask_test.go
@@ -168,6 +168,65 @@ func TestAll(t *testing.T) {
|
||||
assert.NoError(err)
|
||||
})
|
||||
|
||||
t.Run("Sift", func(t *testing.T) {
|
||||
err = db.Put([]byte("toBeSifted"), []byte("siftMe"))
|
||||
assert.NoError(err)
|
||||
err = db.Put([]byte("notToBeSifted"), []byte("dontSiftMe"))
|
||||
assert.NoError(err)
|
||||
err := db.Sift(func(key []byte) (bool, error) {
|
||||
value, err := db.Get(key)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if string(value) == "siftMe" {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
assert.NoError(err)
|
||||
_, err = db.Get([]byte("toBeSifted"))
|
||||
assert.Equal(ErrKeyNotFound, err)
|
||||
_, err = db.Get([]byte("notToBeSifted"))
|
||||
assert.NoError(err)
|
||||
})
|
||||
|
||||
t.Run("SiftScan", func(t *testing.T) {
|
||||
err := db.DeleteAll()
|
||||
assert.NoError(err)
|
||||
err = db.Put([]byte("toBeSifted"), []byte("siftMe"))
|
||||
assert.NoError(err)
|
||||
err = db.Put([]byte("toBeSkipped"), []byte("siftMe"))
|
||||
assert.NoError(err)
|
||||
err = db.Put([]byte("toBeSiftedAsWell"), []byte("siftMe"))
|
||||
assert.NoError(err)
|
||||
err = db.Put([]byte("toBeSiftedButNotReally"), []byte("dontSiftMe"))
|
||||
assert.NoError(err)
|
||||
err = db.SiftScan([]byte("toBeSifted"), func(key []byte) (bool, error) {
|
||||
value, err := db.Get(key)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if string(value) == "siftMe" {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
assert.NoError(err)
|
||||
_, err = db.Get([]byte("toBeSifted"))
|
||||
assert.Equal(ErrKeyNotFound, err)
|
||||
_, err = db.Get([]byte("toBeSiftedAsWell"))
|
||||
assert.Equal(ErrKeyNotFound, err)
|
||||
_, err = db.Get([]byte("toBeSkipped"))
|
||||
assert.NoError(err)
|
||||
_, err = db.Get([]byte("toBeSiftedButNotReally"))
|
||||
assert.NoError(err)
|
||||
})
|
||||
|
||||
t.Run("DeleteAll", func(t *testing.T) {
|
||||
err = db.DeleteAll()
|
||||
assert.NoError(err)
|
||||
})
|
||||
|
||||
t.Run("Close", func(t *testing.T) {
|
||||
err = db.Close()
|
||||
assert.NoError(err)
|
||||
@@ -1655,6 +1714,93 @@ func TestConcurrent(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestSift(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
testdir, err := ioutil.TempDir("", "bitcask")
|
||||
assert.NoError(err)
|
||||
|
||||
var db *Bitcask
|
||||
|
||||
t.Run("Setup", func(t *testing.T) {
|
||||
t.Run("Open", func(t *testing.T) {
|
||||
db, err = Open(testdir)
|
||||
assert.NoError(err)
|
||||
})
|
||||
|
||||
t.Run("Put", func(t *testing.T) {
|
||||
var items = map[string][]byte{
|
||||
"1": []byte("1"),
|
||||
"2": []byte("2"),
|
||||
"3": []byte("3"),
|
||||
"food": []byte("pizza"),
|
||||
"foo": []byte([]byte("foo")),
|
||||
"fooz": []byte("fooz ball"),
|
||||
"hello": []byte("world"),
|
||||
}
|
||||
for k, v := range items {
|
||||
err = db.Put([]byte(k), v)
|
||||
assert.NoError(err)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("SiftErrors", func(t *testing.T) {
|
||||
err = db.Sift(func(key []byte) (bool, error) {
|
||||
return false, ErrMockError
|
||||
})
|
||||
assert.Equal(ErrMockError, err)
|
||||
|
||||
err = db.SiftScan([]byte("fo"), func(key []byte) (bool, error) {
|
||||
return true, ErrMockError
|
||||
})
|
||||
assert.Equal(ErrMockError, err)
|
||||
})
|
||||
}
|
||||
func TestSiftScan(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
testdir, err := ioutil.TempDir("", "bitcask")
|
||||
assert.NoError(err)
|
||||
|
||||
var db *Bitcask
|
||||
|
||||
t.Run("Setup", func(t *testing.T) {
|
||||
t.Run("Open", func(t *testing.T) {
|
||||
db, err = Open(testdir)
|
||||
assert.NoError(err)
|
||||
})
|
||||
|
||||
t.Run("Put", func(t *testing.T) {
|
||||
var items = map[string][]byte{
|
||||
"1": []byte("1"),
|
||||
"2": []byte("2"),
|
||||
"3": []byte("3"),
|
||||
"food": []byte("pizza"),
|
||||
"foo": []byte([]byte("foo")),
|
||||
"fooz": []byte("fooz ball"),
|
||||
"hello": []byte("world"),
|
||||
}
|
||||
for k, v := range items {
|
||||
err = db.Put([]byte(k), v)
|
||||
assert.NoError(err)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("SiftScanErrors", func(t *testing.T) {
|
||||
err = db.SiftScan([]byte("fo"), func(key []byte) (bool, error) {
|
||||
return false, ErrMockError
|
||||
})
|
||||
assert.Equal(ErrMockError, err)
|
||||
|
||||
err = db.SiftScan([]byte("fo"), func(key []byte) (bool, error) {
|
||||
return true, ErrMockError
|
||||
})
|
||||
assert.Equal(ErrMockError, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestScan(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
@@ -1714,6 +1860,146 @@ func TestScan(t *testing.T) {
|
||||
assert.Equal(ErrMockError, err)
|
||||
})
|
||||
}
|
||||
func TestSiftRange(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
testdir, err := ioutil.TempDir("", "bitcask")
|
||||
assert.NoError(err)
|
||||
|
||||
var db *Bitcask
|
||||
|
||||
t.Run("Setup", func(t *testing.T) {
|
||||
t.Run("Open", func(t *testing.T) {
|
||||
db, err = Open(testdir)
|
||||
assert.NoError(err)
|
||||
})
|
||||
|
||||
t.Run("Put", func(t *testing.T) {
|
||||
for i := 1; i < 10; i++ {
|
||||
key := []byte(fmt.Sprintf("foo_%d", i))
|
||||
val := []byte(fmt.Sprintf("%d", i))
|
||||
err = db.Put(key, val)
|
||||
assert.NoError(err)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("SiftRange", func(t *testing.T) {
|
||||
var (
|
||||
vals [][]byte
|
||||
expected = [][]byte{
|
||||
[]byte("1"),
|
||||
[]byte("2"),
|
||||
[]byte("4"),
|
||||
[]byte("5"),
|
||||
[]byte("6"),
|
||||
[]byte("7"),
|
||||
[]byte("8"),
|
||||
[]byte("9"),
|
||||
}
|
||||
)
|
||||
|
||||
err = db.SiftRange([]byte("foo_3"), []byte("foo_7"), func(key []byte) (bool, error) {
|
||||
val, err := db.Get(key)
|
||||
assert.NoError(err)
|
||||
if string(val) == "3" {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
err = db.Fold(func(key []byte) error {
|
||||
val, err := db.Get(key)
|
||||
assert.NoError(err)
|
||||
vals = append(vals, val)
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
_, err = db.Get([]byte("foo_3"))
|
||||
assert.Equal(ErrKeyNotFound, err)
|
||||
vals = SortByteArrays(vals)
|
||||
assert.Equal(expected, vals)
|
||||
})
|
||||
|
||||
t.Run("SiftRangeErrors", func(t *testing.T) {
|
||||
err = db.SiftRange([]byte("foo_3"), []byte("foo_7"), func(key []byte) (bool, error) {
|
||||
return true, ErrMockError
|
||||
})
|
||||
assert.Error(err)
|
||||
assert.Equal(ErrMockError, err)
|
||||
})
|
||||
|
||||
t.Run("InvalidRange", func(t *testing.T) {
|
||||
err = db.SiftRange([]byte("foo_3"), []byte("foo_1"), func(key []byte) (bool, error) {
|
||||
return false, nil
|
||||
})
|
||||
assert.Error(err)
|
||||
assert.Equal(ErrInvalidRange, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestRange(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
testdir, err := ioutil.TempDir("", "bitcask")
|
||||
assert.NoError(err)
|
||||
|
||||
var db *Bitcask
|
||||
|
||||
t.Run("Setup", func(t *testing.T) {
|
||||
t.Run("Open", func(t *testing.T) {
|
||||
db, err = Open(testdir)
|
||||
assert.NoError(err)
|
||||
})
|
||||
|
||||
t.Run("Put", func(t *testing.T) {
|
||||
for i := 1; i < 10; i++ {
|
||||
key := []byte(fmt.Sprintf("foo_%d", i))
|
||||
val := []byte(fmt.Sprintf("%d", i))
|
||||
err = db.Put(key, val)
|
||||
assert.NoError(err)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("Range", func(t *testing.T) {
|
||||
var (
|
||||
vals [][]byte
|
||||
expected = [][]byte{
|
||||
[]byte("3"),
|
||||
[]byte("4"),
|
||||
[]byte("5"),
|
||||
[]byte("6"),
|
||||
[]byte("7"),
|
||||
}
|
||||
)
|
||||
|
||||
err = db.Range([]byte("foo_3"), []byte("foo_7"), func(key []byte) error {
|
||||
val, err := db.Get(key)
|
||||
assert.NoError(err)
|
||||
vals = append(vals, val)
|
||||
return nil
|
||||
})
|
||||
vals = SortByteArrays(vals)
|
||||
assert.Equal(expected, vals)
|
||||
})
|
||||
|
||||
t.Run("RangeErrors", func(t *testing.T) {
|
||||
err = db.Range([]byte("foo_3"), []byte("foo_7"), func(key []byte) error {
|
||||
return ErrMockError
|
||||
})
|
||||
assert.Error(err)
|
||||
assert.Equal(ErrMockError, err)
|
||||
})
|
||||
|
||||
t.Run("InvalidRange", func(t *testing.T) {
|
||||
err = db.Range([]byte("foo_3"), []byte("foo_1"), func(key []byte) error {
|
||||
return nil
|
||||
})
|
||||
assert.Error(err)
|
||||
assert.Equal(ErrInvalidRange, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestLocking(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
@@ -1750,6 +2036,74 @@ func TestLockingAfterMerge(t *testing.T) {
|
||||
assert.Error(err)
|
||||
}
|
||||
|
||||
func TestGetExpiredInsideFold(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
testdir, err := ioutil.TempDir("", "bitcask")
|
||||
assert.NoError(err)
|
||||
|
||||
db, err := Open(testdir)
|
||||
assert.NoError(err)
|
||||
defer db.Close()
|
||||
// Add a node to the tree that won't expire
|
||||
db.Put([]byte("static"), []byte("static"))
|
||||
// Add a node that expires almost immediately to the tree
|
||||
db.PutWithTTL([]byte("shortLived"), []byte("shortLived"), 1*time.Millisecond)
|
||||
db.Put([]byte("skipped"), []byte("skipped"))
|
||||
db.Put([]byte("static2"), []byte("static2"))
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
var arr []string
|
||||
_ = db.Fold(func(key []byte) error {
|
||||
val, err := db.Get(key)
|
||||
switch string(key) {
|
||||
case "skipped":
|
||||
fallthrough
|
||||
case "static2":
|
||||
fallthrough
|
||||
case "static":
|
||||
assert.NoError(err)
|
||||
assert.Equal(string(val), string(key))
|
||||
case "shortLived":
|
||||
assert.Error(err)
|
||||
}
|
||||
arr = append(arr, string(val))
|
||||
return nil
|
||||
})
|
||||
assert.Contains(arr, "skipped")
|
||||
}
|
||||
|
||||
func TestRunGCDeletesAllExpired(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
testdir, err := ioutil.TempDir("", "bitcask")
|
||||
assert.NoError(err)
|
||||
|
||||
db, err := Open(testdir)
|
||||
assert.NoError(err)
|
||||
defer db.Close()
|
||||
|
||||
// Add a node to the tree that won't expire
|
||||
db.Put([]byte("static"), []byte("static"))
|
||||
|
||||
// Add a node that expires almost immediately to the tree
|
||||
db.PutWithTTL([]byte("shortLived"), []byte("shortLived"), 0)
|
||||
db.PutWithTTL([]byte("longLived"), []byte("longLived"), time.Hour)
|
||||
db.PutWithTTL([]byte("longLived2"), []byte("longLived2"), time.Hour)
|
||||
db.PutWithTTL([]byte("shortLived2"), []byte("shortLived2"), 0)
|
||||
db.PutWithTTL([]byte("shortLived3"), []byte("shortLived3"), 0)
|
||||
db.Put([]byte("static2"), []byte("static2"))
|
||||
|
||||
// Sleep a bit and run the Garbage Collector
|
||||
time.Sleep(3 * time.Millisecond)
|
||||
db.RunGC()
|
||||
|
||||
_ = db.Fold(func(key []byte) error {
|
||||
_, err := db.Get(key)
|
||||
assert.NoError(err)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
type benchmarkTestCase struct {
|
||||
name string
|
||||
size int
|
||||
|
||||
61
cmd/bitcask/range.go
Normal file
61
cmd/bitcask/range.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.mills.io/prologic/bitcask"
|
||||
)
|
||||
|
||||
var rangeCmd = &cobra.Command{
|
||||
Use: "range <start> <end>",
|
||||
Aliases: []string{},
|
||||
Short: "Perform a range scan for keys from a start to end key",
|
||||
Long: `This performa a range scan for keys starting with the given start
|
||||
key and ending with the end key. This uses a Trie to search for matching keys
|
||||
within the range and returns all matched keys.`,
|
||||
Args: cobra.ExactArgs(2),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
path := viper.GetString("path")
|
||||
|
||||
start := args[0]
|
||||
end := args[1]
|
||||
|
||||
os.Exit(_range(path, start, end))
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
RootCmd.AddCommand(rangeCmd)
|
||||
}
|
||||
|
||||
func _range(path, start, end string) int {
|
||||
db, err := bitcask.Open(path)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("error opening database")
|
||||
return 1
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
err = db.Range([]byte(start), []byte(end), func(key []byte) error {
|
||||
value, err := db.Get(key)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("error reading key")
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("%s\n", string(value))
|
||||
log.WithField("key", key).WithField("value", value).Debug("key/value")
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
log.WithError(err).Error("error ranging over keys")
|
||||
return 1
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
@@ -84,10 +84,41 @@ func (s *server) handleGet(cmd redcon.Command, conn redcon.Conn) {
|
||||
}
|
||||
|
||||
func (s *server) handleKeys(cmd redcon.Command, conn redcon.Conn) {
|
||||
conn.WriteArray(s.db.Len())
|
||||
for key := range s.db.Keys() {
|
||||
conn.WriteBulk(key)
|
||||
if len(cmd.Args) != 2 {
|
||||
conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
|
||||
return
|
||||
}
|
||||
|
||||
pattern := string(cmd.Args[1])
|
||||
|
||||
// Fast-track condition for improved speed
|
||||
if pattern == "*" {
|
||||
conn.WriteArray(s.db.Len())
|
||||
for key := range s.db.Keys() {
|
||||
conn.WriteBulk(key)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Prefix handling
|
||||
if strings.Count(pattern, "*") == 1 && strings.HasSuffix(pattern, "*") {
|
||||
prefix := strings.ReplaceAll(pattern, "*", "")
|
||||
count := 0
|
||||
keys := make([][]byte, 0)
|
||||
s.db.Scan([]byte(prefix), func(key []byte) error {
|
||||
keys = append(keys, key)
|
||||
count++
|
||||
return nil
|
||||
})
|
||||
conn.WriteArray(count)
|
||||
for _, key := range keys {
|
||||
conn.WriteBulk(key)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// No results means empty array
|
||||
conn.WriteArray(0)
|
||||
}
|
||||
|
||||
func (s *server) handleExists(cmd redcon.Command, conn redcon.Conn) {
|
||||
|
||||
102
cmd/bitcaskd/server_test.go
Normal file
102
cmd/bitcaskd/server_test.go
Normal file
@@ -0,0 +1,102 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/tidwall/redcon"
|
||||
)
|
||||
|
||||
func TestHandleKeys(t *testing.T) {
|
||||
s, err := newServer(":61234", "./test.db")
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create server: %v", err)
|
||||
}
|
||||
s.db.Put([]byte("foo"), []byte("bar"))
|
||||
testCases := []TestCase{
|
||||
{
|
||||
Command: redcon.Command{
|
||||
Raw: []byte("KEYS *"),
|
||||
Args: [][]byte{[]byte("KEYS"), []byte("*")},
|
||||
},
|
||||
Expected: "1,foo",
|
||||
},
|
||||
{
|
||||
Command: redcon.Command{
|
||||
Raw: []byte("KEYS fo*"),
|
||||
Args: [][]byte{[]byte("KEYS"), []byte("fo*")},
|
||||
},
|
||||
Expected: "1,foo",
|
||||
},
|
||||
{
|
||||
Command: redcon.Command{
|
||||
Raw: []byte("KEYS ba*"),
|
||||
Args: [][]byte{[]byte("KEYS"), []byte("ba*")},
|
||||
},
|
||||
Expected: "0",
|
||||
},
|
||||
{
|
||||
Command: redcon.Command{
|
||||
Raw: []byte("KEYS *oo"),
|
||||
Args: [][]byte{[]byte("KEYS"), []byte("*oo")},
|
||||
},
|
||||
Expected: "0",
|
||||
},
|
||||
}
|
||||
for _, testCase := range testCases {
|
||||
conn := DummyConn{}
|
||||
s.handleKeys(testCase.Command, &conn)
|
||||
if testCase.Expected != conn.Result {
|
||||
t.Fatalf("s.handleKeys failed: expected '%s', got '%s'", testCase.Expected, conn.Result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type TestCase struct {
|
||||
Command redcon.Command
|
||||
Expected string
|
||||
}
|
||||
|
||||
type DummyConn struct {
|
||||
Result string
|
||||
}
|
||||
|
||||
func (dc *DummyConn) RemoteAddr() string {
|
||||
return ""
|
||||
}
|
||||
func (dc *DummyConn) Close() error {
|
||||
return nil
|
||||
}
|
||||
func (dc *DummyConn) WriteError(msg string) {}
|
||||
func (dc *DummyConn) WriteString(str string) {}
|
||||
func (dc *DummyConn) WriteBulk(bulk []byte) {
|
||||
dc.Result += "," + string(bulk)
|
||||
}
|
||||
func (dc *DummyConn) WriteBulkString(bulk string) {}
|
||||
func (dc *DummyConn) WriteInt(num int) {}
|
||||
func (dc *DummyConn) WriteInt64(num int64) {}
|
||||
func (dc *DummyConn) WriteUint64(num uint64) {}
|
||||
func (dc *DummyConn) WriteArray(count int) {
|
||||
dc.Result = strconv.Itoa(count)
|
||||
}
|
||||
func (dc *DummyConn) WriteNull() {}
|
||||
func (dc *DummyConn) WriteRaw(data []byte) {}
|
||||
func (dc *DummyConn) WriteAny(any interface{}) {}
|
||||
func (dc *DummyConn) Context() interface{} {
|
||||
return nil
|
||||
}
|
||||
func (dc *DummyConn) SetContext(v interface{}) {}
|
||||
func (dc *DummyConn) SetReadBuffer(bytes int) {}
|
||||
func (dc *DummyConn) Detach() redcon.DetachedConn {
|
||||
return nil
|
||||
}
|
||||
func (dc *DummyConn) ReadPipeline() []redcon.Command {
|
||||
return nil
|
||||
}
|
||||
func (dc *DummyConn) PeekPipeline() []redcon.Command {
|
||||
return nil
|
||||
}
|
||||
func (dc *DummyConn) NetConn() net.Conn {
|
||||
return nil
|
||||
}
|
||||
77
errors.go
Normal file
77
errors.go
Normal file
@@ -0,0 +1,77 @@
|
||||
package bitcask
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrKeyNotFound is the error returned when a key is not found
|
||||
ErrKeyNotFound = errors.New("error: key not found")
|
||||
|
||||
// ErrKeyTooLarge is the error returned for a key that exceeds the
|
||||
// maximum allowed key size (configured with WithMaxKeySize).
|
||||
ErrKeyTooLarge = errors.New("error: key too large")
|
||||
|
||||
// ErrKeyExpired is the error returned when a key is queried which has
|
||||
// already expired (due to ttl)
|
||||
ErrKeyExpired = errors.New("error: key expired")
|
||||
|
||||
// ErrEmptyKey is the error returned for a value with an empty key.
|
||||
ErrEmptyKey = errors.New("error: empty key")
|
||||
|
||||
// ErrValueTooLarge is the error returned for a value that exceeds the
|
||||
// maximum allowed value size (configured with WithMaxValueSize).
|
||||
ErrValueTooLarge = errors.New("error: value too large")
|
||||
|
||||
// ErrChecksumFailed is the error returned if a key/value retrieved does
|
||||
// not match its CRC checksum
|
||||
ErrChecksumFailed = errors.New("error: checksum failed")
|
||||
|
||||
// ErrDatabaseLocked is the error returned if the database is locked
|
||||
// (typically opened by another process)
|
||||
ErrDatabaseLocked = errors.New("error: database locked")
|
||||
|
||||
// ErrInvalidRange is the error returned when the range scan is invalid
|
||||
ErrInvalidRange = errors.New("error: invalid range")
|
||||
|
||||
// ErrInvalidVersion is the error returned when the database version is invalid
|
||||
ErrInvalidVersion = errors.New("error: invalid db version")
|
||||
|
||||
// ErrMergeInProgress is the error returned if merge is called when already a merge
|
||||
// is in progress
|
||||
ErrMergeInProgress = errors.New("error: merge already in progress")
|
||||
)
|
||||
|
||||
// ErrBadConfig is the error returned on failure to load the database config
|
||||
type ErrBadConfig struct {
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *ErrBadConfig) Is(target error) bool {
|
||||
if _, ok := target.(*ErrBadConfig); ok {
|
||||
return true
|
||||
}
|
||||
return errors.Is(e.Err, target)
|
||||
}
|
||||
func (e *ErrBadConfig) Unwrap() error { return e.Err }
|
||||
func (e *ErrBadConfig) Error() string {
|
||||
return fmt.Sprintf("error reading config.json: %s", e.Err)
|
||||
}
|
||||
|
||||
// ErrBadMetadata is the error returned on failure to load the database metadata
|
||||
type ErrBadMetadata struct {
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e *ErrBadMetadata) Is(target error) bool {
|
||||
if _, ok := target.(*ErrBadMetadata); ok {
|
||||
return true
|
||||
}
|
||||
return errors.Is(e.Err, target)
|
||||
}
|
||||
|
||||
func (e *ErrBadMetadata) Unwrap() error { return e.Err }
|
||||
func (e *ErrBadMetadata) Error() string {
|
||||
return fmt.Sprintf("error reading meta.json: %s", e.Err)
|
||||
}
|
||||
5
go.mod
5
go.mod
@@ -1,12 +1,13 @@
|
||||
module git.mills.io/prologic/bitcask
|
||||
|
||||
go 1.13
|
||||
go 1.16
|
||||
|
||||
require (
|
||||
github.com/abcum/lcp v0.0.0-20201209214815-7a3f3840be81
|
||||
github.com/gofrs/flock v0.8.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/plar/go-adaptive-radix-tree v1.0.4
|
||||
github.com/sirupsen/logrus v1.8.1
|
||||
github.com/sirupsen/logrus v1.8.3
|
||||
github.com/spf13/cobra v0.0.7
|
||||
github.com/spf13/pflag v1.0.5
|
||||
github.com/spf13/viper v1.8.1
|
||||
|
||||
16
go.sum
16
go.sum
@@ -40,6 +40,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/abcum/lcp v0.0.0-20201209214815-7a3f3840be81 h1:uHogIJ9bXH75ZYrXnVShHIyywFiUZ7OOabwd9Sfd8rw=
|
||||
github.com/abcum/lcp v0.0.0-20201209214815-7a3f3840be81/go.mod h1:6ZvnjTZX1LNo1oLpfaJK8h+MXqHxcBFBIwkgsv+xlv0=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
@@ -160,8 +162,9 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
|
||||
@@ -264,10 +267,11 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
|
||||
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
|
||||
github.com/sirupsen/logrus v1.8.3 h1:DBBfY8eMYazKEJHb3JKpSPfpgd2mBCoNFlQx6C5fftU=
|
||||
github.com/sirupsen/logrus v1.8.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
|
||||
github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
|
||||
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
|
||||
@@ -457,7 +461,6 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@@ -485,8 +488,9 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
||||
@@ -6,9 +6,9 @@ import (
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"git.mills.io/prologic/bitcask/internal"
|
||||
"git.mills.io/prologic/bitcask/internal/data/codec"
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/exp/mmap"
|
||||
)
|
||||
|
||||
@@ -74,9 +74,11 @@ func NewDatafile(path string, id int, readonly bool, maxKeySize uint32, maxValue
|
||||
return nil, errors.Wrap(err, "error calling Stat()")
|
||||
}
|
||||
|
||||
ra, err = mmap.Open(fn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
if readonly {
|
||||
ra, err = mmap.Open(fn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
offset := stat.Size()
|
||||
@@ -107,7 +109,9 @@ func (df *datafile) Name() string {
|
||||
|
||||
func (df *datafile) Close() error {
|
||||
defer func() {
|
||||
df.ra.Close()
|
||||
if df.ra != nil {
|
||||
df.ra.Close()
|
||||
}
|
||||
df.r.Close()
|
||||
}()
|
||||
|
||||
@@ -155,7 +159,10 @@ func (df *datafile) ReadAt(index, size int64) (e internal.Entry, err error) {
|
||||
|
||||
b := make([]byte, size)
|
||||
|
||||
if df.w == nil {
|
||||
df.RLock()
|
||||
defer df.RUnlock()
|
||||
|
||||
if df.ra != nil {
|
||||
n, err = df.ra.ReadAt(b, index)
|
||||
} else {
|
||||
n, err = df.r.ReadAt(b, index)
|
||||
|
||||
@@ -2,17 +2,49 @@ package internal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultVersion = "0.0.0"
|
||||
defaultCommit = "HEAD"
|
||||
defaultBuild = "0000-01-01:00:00+00:00"
|
||||
)
|
||||
|
||||
var (
|
||||
// Version release version
|
||||
Version = "0.0.1"
|
||||
// Version is the tagged release version in the form <major>.<minor>.<patch>
|
||||
// following semantic versioning and is overwritten by the build system.
|
||||
Version = defaultVersion
|
||||
|
||||
// Commit will be overwritten automatically by the build system
|
||||
Commit = "HEAD"
|
||||
// Commit is the commit sha of the build (normally from Git) and is overwritten
|
||||
// by the build system.
|
||||
Commit = defaultCommit
|
||||
|
||||
// Build is the date and time of the build as an RFC3339 formatted string
|
||||
// and is overwritten by the build system.
|
||||
Build = defaultBuild
|
||||
)
|
||||
|
||||
// FullVersion returns the full version and commit hash
|
||||
// FullVersion display the full version and build
|
||||
func FullVersion() string {
|
||||
return fmt.Sprintf("%s@%s", Version, Commit)
|
||||
var sb strings.Builder
|
||||
|
||||
isDefault := Version == defaultVersion && Commit == defaultCommit && Build == defaultBuild
|
||||
|
||||
if !isDefault {
|
||||
sb.WriteString(fmt.Sprintf("%s@%s %s", Version, Commit, Build))
|
||||
}
|
||||
|
||||
if info, ok := debug.ReadBuildInfo(); ok {
|
||||
if isDefault {
|
||||
sb.WriteString(fmt.Sprintf(" %s", info.Main.Version))
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf(" %s", info.GoVersion))
|
||||
if info.Main.Sum != "" {
|
||||
sb.WriteString(fmt.Sprintf(" %s", info.Main.Sum))
|
||||
}
|
||||
}
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ echo "Releasing ${TAG} ..."
|
||||
git-chglog --next-tag="${TAG}" --output CHANGELOG.md
|
||||
git commit -a -m "Update CHANGELOG for ${TAG}"
|
||||
git tag -a -s -m "Release ${TAG}" "${TAG}"
|
||||
git push --tags
|
||||
git push && git push --tags
|
||||
goreleaser release \
|
||||
--rm-dist \
|
||||
--release-notes <(git-chglog "${TAG}" | tail -n+5)
|
||||
|
||||
Reference in New Issue
Block a user