taigrr/bitcask
1
0
Fork 0
mirror of https://github.com/taigrr/bitcask synced 2025-01-18 04:03:17 -08:00
Code Issues Projects Releases Wiki Activity

Optimized and increased read performance by ~2-3x by memory mapping the read-only datafiles

Browse Source
This commit is contained in:
James Mills 2019-03-23 13:35:55 +10:00
parent 711d08ce91
commit 53dc013215
No known key found for this signature in database
GPG Key ID: AC4C014F1440EBD6
4 changed files with 48 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
README.md
View File

@ -94,32 +94,32 @@ Benchmarks run on a 11" Macbook with a 1.4Ghz Intel Core i7:
```
$ make bench
...
BenchmarkGet/128B-4 300000 3737 ns/op 1632 B/op 16 allocs/op
BenchmarkGet/256B-4 300000 4183 ns/op 2016 B/op 16 allocs/op
BenchmarkGet/512B-4 300000 4295 ns/op 2848 B/op 16 allocs/op
BenchmarkGet/1K-4 300000 4455 ns/op 4512 B/op 16 allocs/op
BenchmarkGet/2K-4 300000 5536 ns/op 7841 B/op 16 allocs/op
BenchmarkGet/4K-4 200000 7101 ns/op 15010 B/op 16 allocs/op
BenchmarkGet/8K-4 200000 10664 ns/op 28325 B/op 16 allocs/op
BenchmarkGet/16K-4 100000 18173 ns/op 54442 B/op 16 allocs/op
BenchmarkGet/32K-4 50000 33081 ns/op 115893 B/op 16 allocs/op
BenchmarkGet/128B-4 500000 2537 ns/op 672 B/op 7 allocs/op
BenchmarkGet/256B-4 500000 2629 ns/op 1056 B/op 7 allocs/op
BenchmarkGet/512B-4 500000 2773 ns/op 1888 B/op 7 allocs/op
BenchmarkGet/1K-4 500000 3202 ns/op 3552 B/op 7 allocs/op
BenchmarkGet/2K-4 300000 3904 ns/op 6880 B/op 7 allocs/op
BenchmarkGet/4K-4 300000 5678 ns/op 14048 B/op 7 allocs/op
BenchmarkGet/8K-4 200000 8948 ns/op 27360 B/op 7 allocs/op
BenchmarkGet/16K-4 100000 14635 ns/op 53472 B/op 7 allocs/op
BenchmarkGet/32K-4 50000 28292 ns/op 114912 B/op 7 allocs/op
BenchmarkPut/128B-4 200000 7967 ns/op 409 B/op 6 allocs/op
BenchmarkPut/256B-4 200000 8563 ns/op 538 B/op 6 allocs/op
BenchmarkPut/512B-4 200000 9678 ns/op 829 B/op 6 allocs/op
BenchmarkPut/1K-4 200000 12786 ns/op 1410 B/op 6 allocs/op
BenchmarkPut/2K-4 100000 18582 ns/op 2572 B/op 6 allocs/op
BenchmarkPut/4K-4 50000 35335 ns/op 5151 B/op 6 allocs/op
BenchmarkPut/8K-4 30000 56047 ns/op 9797 B/op 6 allocs/op
BenchmarkPut/16K-4 20000 86137 ns/op 18834 B/op 6 allocs/op
BenchmarkPut/32K-4 10000 140162 ns/op 41517 B/op 6 allocs/op
BenchmarkPut/128B-4 200000 8173 ns/op 409 B/op 6 allocs/op
BenchmarkPut/256B-4 200000 8404 ns/op 538 B/op 6 allocs/op
BenchmarkPut/512B-4 200000 9741 ns/op 829 B/op 6 allocs/op
BenchmarkPut/1K-4 100000 13118 ns/op 1411 B/op 6 allocs/op
BenchmarkPut/2K-4 100000 17982 ns/op 2573 B/op 6 allocs/op
BenchmarkPut/4K-4 50000 35477 ns/op 5154 B/op 6 allocs/op
BenchmarkPut/8K-4 30000 54021 ns/op 9804 B/op 6 allocs/op
BenchmarkPut/16K-4 20000 96551 ns/op 18849 B/op 6 allocs/op
BenchmarkPut/32K-4 10000 129957 ns/op 41561 B/op 7 allocs/op
BenchmarkScan-4 1000000 1885 ns/op 493 B/op 25 allocs/op
BenchmarkScan-4 1000000 2011 ns/op 493 B/op 25 allocs/op
```
For 128B values:
* ~270,000 reads/sec
* ~400,000 reads/sec
* ~130,000 writes/sec
The full benchmark above shows linear performance as you increase key/value sizes.

1
go.mod
View File

@ -21,6 +21,7 @@ require (
github.com/tidwall/redcon v1.0.0
github.com/ugorji/go/codec v0.0.0-20190320090025-2dc34c0b8780 // indirect
golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576 // indirect
golang.org/x/exp v0.0.0-20190321205749-f0864edee7f3
golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53 // indirect
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 // indirect
golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc // indirect

7
go.sum
View File

@ -1,5 +1,6 @@
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@ -84,6 +85,10 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576 h1:aUX/1G2gFSs4AsJJg2cL3HuoRhCSCz733FE5GUSuaT4=
golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/exp v0.0.0-20190321205749-f0864edee7f3 h1:Ep4L2ibjtJcW6IP73KbcJAU0cpNKsLNSSP2jE1xlCys=
golang.org/x/exp v0.0.0-20190321205749-f0864edee7f3/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
@ -93,12 +98,14 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc h1:4gbWbmmPFp4ySWICouJl6emP0MyS31yy9SrTlAGFT+g=
golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190321232350-e250d351ecad/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

23
internal/datafile.go
View File

@ -8,7 +8,7 @@ import (
"sync"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/exp/mmap"
pb "github.com/prologic/bitcask/internal/proto"
"github.com/prologic/bitcask/internal/streampb"
@ -28,6 +28,7 @@ type Datafile struct {
id int
r *os.File
ra *mmap.ReaderAt
w *os.File
offset int64
dec *streampb.Decoder
@ -37,6 +38,7 @@ type Datafile struct {
func NewDatafile(path string, id int, readonly bool) (*Datafile, error) {
var (
r *os.File
ra *mmap.ReaderAt
w *os.File
err error
)
@ -59,6 +61,11 @@ func NewDatafile(path string, id int, readonly bool) (*Datafile, error) {
return nil, errors.Wrap(err, "error calling Stat()")
}
ra, err = mmap.Open(fn)
if err != nil {
return nil, err
}
offset := stat.Size()
dec := streampb.NewDecoder(r)
@ -67,6 +74,7 @@ func NewDatafile(path string, id int, readonly bool) (*Datafile, error) {
return &Datafile{
id: id,
r: r,
ra: ra,
w: w,
offset: offset,
dec: dec,
@ -84,6 +92,10 @@ func (df *Datafile) Name() string {
func (df *Datafile) Close() error {
if df.w == nil {
err := df.ra.Close()
if err != nil {
return err
}
return df.r.Close()
}
@ -120,10 +132,15 @@ func (df *Datafile) Read() (e pb.Entry, n int64, err error) {
}
func (df *Datafile) ReadAt(index, size int64) (e pb.Entry, err error) {
log.WithField("index", index).WithField("size", size).Debug("ReadAt")
var n int
b := make([]byte, size)
n, err := df.r.ReadAt(b, index)
if df.w == nil {
n, err = df.ra.ReadAt(b, index)
} else {
n, err = df.r.ReadAt(b, index)
}
if err != nil {
return
}