mirror of
https://github.com/taigrr/bitcask
synced 2025-01-18 04:03:17 -08:00
Add LFU w/ Tests
This commit is contained in:
Haleem Assal
parent
f397bec88f
commit
f2a0e2cacd
15
bitcask.go
15
bitcask.go
@ -12,6 +12,7 @@ import (
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
gcache "github.com/bluele/gcache"
|
||||
art "github.com/plar/go-adaptive-radix-tree"
|
||||
"github.com/prologic/bitcask/flock"
|
||||
"github.com/prologic/bitcask/internal"
|
||||
@ -71,6 +72,7 @@ type Bitcask struct {
|
||||
indexer index.Indexer
|
||||
metadata *metadata.MetaData
|
||||
isMerging bool
|
||||
gc gcache.Cache
|
||||
}
|
||||
|
||||
// Stats is a struct returned by Stats() on an open Bitcask instance
|
||||
@ -163,6 +165,12 @@ func (b *Bitcask) get(key []byte) ([]byte, error) {
|
||||
df = b.curr
|
||||
} else {
|
||||
df = b.datafiles[item.FileID]
|
||||
|
||||
// reopen file and add to lfu
|
||||
if !b.gc.Has(item.FileID) {
|
||||
df.Open()
|
||||
b.gc.Set(item.FileID, df)
|
||||
}
|
||||
}
|
||||
|
||||
e, err := df.ReadAt(item.Offset, item.Size)
|
||||
@ -337,6 +345,7 @@ func (b *Bitcask) put(key, value []byte) (int64, int64, error) {
|
||||
}
|
||||
|
||||
b.datafiles[id] = df
|
||||
b.gc.Set(id, df)
|
||||
|
||||
id = b.curr.FileID() + 1
|
||||
curr, err := data.NewDatafile(b.path, id, false, b.config.MaxKeySize, b.config.MaxValueSize, b.config.FileFileModeBeforeUmask)
|
||||
@ -568,6 +577,11 @@ func Open(path string, options ...Option) (*Bitcask, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
gc := gcache.New(250).LFU().EvictedFunc(func(key, value interface{}) {
|
||||
datafile := value.(data.Datafile)
|
||||
datafile.Close()
|
||||
}).Build()
|
||||
|
||||
bitcask := &Bitcask{
|
||||
Flock: flock.New(filepath.Join(path, lockfile)),
|
||||
config: cfg,
|
||||
@ -575,6 +589,7 @@ func Open(path string, options ...Option) (*Bitcask, error) {
|
||||
path: path,
|
||||
indexer: index.NewIndexer(),
|
||||
metadata: meta,
|
||||
gc: gc,
|
||||
}
|
||||
|
||||
locked, err := bitcask.Flock.TryLock()
|
||||
|
||||
@ -1648,6 +1648,35 @@ func TestLockingAfterMerge(t *testing.T) {
|
||||
assert.Error(err)
|
||||
}
|
||||
|
||||
func TestLFU(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
testdir, err := ioutil.TempDir("", "bitcask")
|
||||
assert.NoError(err)
|
||||
|
||||
db, err := Open(testdir)
|
||||
assert.NoError(err)
|
||||
defer db.Close()
|
||||
|
||||
_, err = Open(testdir)
|
||||
assert.Error(err)
|
||||
|
||||
// LFU size of 250
|
||||
for i := 0; i < 1000; i++ {
|
||||
key := []byte(string(i))
|
||||
value := []byte(string(i))
|
||||
err = db.Put(key, value)
|
||||
assert.NoError(err)
|
||||
}
|
||||
|
||||
for i := 0; i < 1000; i++ {
|
||||
key := []byte(string(i))
|
||||
val, err := db.Get(key)
|
||||
assert.NoError(err)
|
||||
assert.Equal([]byte(string(i)), val)
|
||||
}
|
||||
}
|
||||
|
||||
type benchmarkTestCase struct {
|
||||
name string
|
||||
size int
|
||||
|
||||
2
go.mod
2
go.mod
@ -3,6 +3,7 @@ module github.com/prologic/bitcask
|
||||
go 1.13
|
||||
|
||||
require (
|
||||
github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833
|
||||
github.com/pelletier/go-toml v1.6.0 // indirect
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/plar/go-adaptive-radix-tree v1.0.4
|
||||
@ -16,6 +17,7 @@ require (
|
||||
github.com/stretchr/objx v0.2.0 // indirect
|
||||
github.com/stretchr/testify v1.6.1
|
||||
github.com/tidwall/redcon v1.4.0
|
||||
github.com/vektra/mockery v1.1.2 // indirect
|
||||
golang.org/x/exp v0.0.0-20200228211341-fcea875c7e85
|
||||
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527
|
||||
gopkg.in/ini.v1 v1.53.0 // indirect
|
||||
|
||||
14
go.sum
14
go.sum
@ -25,6 +25,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
|
||||
github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833 h1:yCfXxYaelOyqnia8F/Yng47qhmfC9nKTRIbYRrRueq4=
|
||||
github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833/go.mod h1:8c4/i2VlovMO2gBnHGQPN5EJw+H0lx1u/5p+cgsXtCk=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
|
||||
@ -207,8 +209,11 @@ github.com/tidwall/redcon v1.4.0 h1:y2PmDD55STRdy4S98qP/Dn+gZG+cPVvIDi9BJV2aOwA=
|
||||
github.com/tidwall/redcon v1.4.0/go.mod h1:IGzxyoKE3Ea5AWIXo/ZHP+hzY8sWXaMKr7KlFgcWSZU=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
|
||||
github.com/vektra/mockery v1.1.2 h1:uc0Yn67rJpjt8U/mAZimdCKn9AeA97BOkjpmtBSlfP4=
|
||||
github.com/vektra/mockery v1.1.2/go.mod h1:VcfZjKaFOPO+MpN4ZvwPjs4c48lkq1o3Ym8yHZJu0jU=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
@ -243,6 +248,8 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG
|
||||
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@ -258,6 +265,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
|
||||
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@ -266,6 +274,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@ -306,9 +315,14 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn
|
||||
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200323144430-8dcfad9e016e h1:ssd5ulOvVWlh4kDSUF2SqzmMeWfjmwDXM+uGw/aQjRE=
|
||||
golang.org/x/tools v0.0.0-20200323144430-8dcfad9e016e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
|
||||
@ -25,6 +25,7 @@ var (
|
||||
type Datafile interface {
|
||||
FileID() int
|
||||
Name() string
|
||||
Open() error
|
||||
Close() error
|
||||
Sync() error
|
||||
Size() int64
|
||||
@ -97,6 +98,22 @@ func NewDatafile(path string, id int, readonly bool, maxKeySize uint32, maxValue
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (df *datafile) Open() error {
|
||||
var err error
|
||||
|
||||
df.r, err = os.Open(df.Name())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
df.ra, err = mmap.Open(df.Name())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (df *datafile) FileID() int {
|
||||
return df.id
|
||||
}
|
||||
|
||||
@ -2,8 +2,10 @@
|
||||
|
||||
package mocks
|
||||
|
||||
import internal "github.com/prologic/bitcask/internal"
|
||||
import mock "github.com/stretchr/testify/mock"
|
||||
import (
|
||||
internal "github.com/prologic/bitcask/internal"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// Datafile is an autogenerated mock type for the Datafile type
|
||||
type Datafile struct {
|
||||
@ -52,6 +54,20 @@ func (_m *Datafile) Name() string {
|
||||
return r0
|
||||
}
|
||||
|
||||
// Open provides a mock function with given fields:
|
||||
func (_m *Datafile) Open() error {
|
||||
ret := _m.Called()
|
||||
|
||||
var r0 error
|
||||
if rf, ok := ret.Get(0).(func() error); ok {
|
||||
r0 = rf()
|
||||
} else {
|
||||
r0 = ret.Error(0)
|
||||
}
|
||||
|
||||
return r0
|
||||
}
|
||||
|
||||
// Read provides a mock function with given fields:
|
||||
func (_m *Datafile) Read() (internal.Entry, int64, error) {
|
||||
ret := _m.Called()
|
||||
|
||||
@ -2,9 +2,11 @@
|
||||
|
||||
package mocks
|
||||
|
||||
import art "github.com/plar/go-adaptive-radix-tree"
|
||||
import (
|
||||
art "github.com/plar/go-adaptive-radix-tree"
|
||||
|
||||
import mock "github.com/stretchr/testify/mock"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
)
|
||||
|
||||
// Indexer is an autogenerated mock type for the Indexer type
|
||||
type Indexer struct {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user