From f2a0e2cacdb1311dcd19f993dd7e693e116e59e8 Mon Sep 17 00:00:00 2001 From: Haleem Assal Date: Mon, 21 Dec 2020 02:07:23 -0400 Subject: [PATCH] Add LFU w/ Tests --- bitcask.go | 15 +++++++++++++++ bitcask_test.go | 29 +++++++++++++++++++++++++++++ go.mod | 2 ++ go.sum | 14 ++++++++++++++ internal/data/datafile.go | 17 +++++++++++++++++ internal/mocks/datafile.go | 20 ++++++++++++++++++-- internal/mocks/indexer.go | 6 ++++-- 7 files changed, 99 insertions(+), 4 deletions(-) diff --git a/bitcask.go b/bitcask.go index 4312001..eba55dc 100644 --- a/bitcask.go +++ b/bitcask.go @@ -12,6 +12,7 @@ import ( "sort" "sync" + gcache "github.com/bluele/gcache" art "github.com/plar/go-adaptive-radix-tree" "github.com/prologic/bitcask/flock" "github.com/prologic/bitcask/internal" @@ -71,6 +72,7 @@ type Bitcask struct { indexer index.Indexer metadata *metadata.MetaData isMerging bool + gc gcache.Cache } // Stats is a struct returned by Stats() on an open Bitcask instance @@ -163,6 +165,12 @@ func (b *Bitcask) get(key []byte) ([]byte, error) { df = b.curr } else { df = b.datafiles[item.FileID] + + // reopen file and add to lfu + if !b.gc.Has(item.FileID) { + df.Open() + b.gc.Set(item.FileID, df) + } } e, err := df.ReadAt(item.Offset, item.Size) @@ -337,6 +345,7 @@ func (b *Bitcask) put(key, value []byte) (int64, int64, error) { } b.datafiles[id] = df + b.gc.Set(id, df) id = b.curr.FileID() + 1 curr, err := data.NewDatafile(b.path, id, false, b.config.MaxKeySize, b.config.MaxValueSize, b.config.FileFileModeBeforeUmask) @@ -568,6 +577,11 @@ func Open(path string, options ...Option) (*Bitcask, error) { return nil, err } + gc := gcache.New(250).LFU().EvictedFunc(func(key, value interface{}) { + datafile := value.(data.Datafile) + datafile.Close() + }).Build() + bitcask := &Bitcask{ Flock: flock.New(filepath.Join(path, lockfile)), config: cfg, @@ -575,6 +589,7 @@ func Open(path string, options ...Option) (*Bitcask, error) { path: path, indexer: index.NewIndexer(), metadata: meta, + gc: gc, } locked, err := bitcask.Flock.TryLock() diff --git a/bitcask_test.go b/bitcask_test.go index 45e35be..da1a0e7 100644 --- a/bitcask_test.go +++ b/bitcask_test.go @@ -1648,6 +1648,35 @@ func TestLockingAfterMerge(t *testing.T) { assert.Error(err) } +func TestLFU(t *testing.T) { + assert := assert.New(t) + + testdir, err := ioutil.TempDir("", "bitcask") + assert.NoError(err) + + db, err := Open(testdir) + assert.NoError(err) + defer db.Close() + + _, err = Open(testdir) + assert.Error(err) + + // LFU size of 250 + for i := 0; i < 1000; i++ { + key := []byte(string(i)) + value := []byte(string(i)) + err = db.Put(key, value) + assert.NoError(err) + } + + for i := 0; i < 1000; i++ { + key := []byte(string(i)) + val, err := db.Get(key) + assert.NoError(err) + assert.Equal([]byte(string(i)), val) + } +} + type benchmarkTestCase struct { name string size int diff --git a/go.mod b/go.mod index 890ec3c..a144949 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/prologic/bitcask go 1.13 require ( + github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833 github.com/pelletier/go-toml v1.6.0 // indirect github.com/pkg/errors v0.9.1 github.com/plar/go-adaptive-radix-tree v1.0.4 @@ -16,6 +17,7 @@ require ( github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.6.1 github.com/tidwall/redcon v1.4.0 + github.com/vektra/mockery v1.1.2 // indirect golang.org/x/exp v0.0.0-20200228211341-fcea875c7e85 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 gopkg.in/ini.v1 v1.53.0 // indirect diff --git a/go.sum b/go.sum index 8a9f85f..ceef12e 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833 h1:yCfXxYaelOyqnia8F/Yng47qhmfC9nKTRIbYRrRueq4= +github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833/go.mod h1:8c4/i2VlovMO2gBnHGQPN5EJw+H0lx1u/5p+cgsXtCk= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -207,8 +209,11 @@ github.com/tidwall/redcon v1.4.0 h1:y2PmDD55STRdy4S98qP/Dn+gZG+cPVvIDi9BJV2aOwA= github.com/tidwall/redcon v1.4.0/go.mod h1:IGzxyoKE3Ea5AWIXo/ZHP+hzY8sWXaMKr7KlFgcWSZU= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/vektra/mockery v1.1.2 h1:uc0Yn67rJpjt8U/mAZimdCKn9AeA97BOkjpmtBSlfP4= +github.com/vektra/mockery v1.1.2/go.mod h1:VcfZjKaFOPO+MpN4ZvwPjs4c48lkq1o3Ym8yHZJu0jU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -243,6 +248,8 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -258,6 +265,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -266,6 +274,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -306,9 +315,14 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200323144430-8dcfad9e016e h1:ssd5ulOvVWlh4kDSUF2SqzmMeWfjmwDXM+uGw/aQjRE= +golang.org/x/tools v0.0.0-20200323144430-8dcfad9e016e/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= diff --git a/internal/data/datafile.go b/internal/data/datafile.go index 1679dcc..2bcb97b 100644 --- a/internal/data/datafile.go +++ b/internal/data/datafile.go @@ -25,6 +25,7 @@ var ( type Datafile interface { FileID() int Name() string + Open() error Close() error Sync() error Size() int64 @@ -97,6 +98,22 @@ func NewDatafile(path string, id int, readonly bool, maxKeySize uint32, maxValue }, nil } +func (df *datafile) Open() error { + var err error + + df.r, err = os.Open(df.Name()) + if err != nil { + return err + } + + df.ra, err = mmap.Open(df.Name()) + if err != nil { + return err + } + + return nil +} + func (df *datafile) FileID() int { return df.id } diff --git a/internal/mocks/datafile.go b/internal/mocks/datafile.go index 7cbcddf..1cb182c 100644 --- a/internal/mocks/datafile.go +++ b/internal/mocks/datafile.go @@ -2,8 +2,10 @@ package mocks -import internal "github.com/prologic/bitcask/internal" -import mock "github.com/stretchr/testify/mock" +import ( + internal "github.com/prologic/bitcask/internal" + mock "github.com/stretchr/testify/mock" +) // Datafile is an autogenerated mock type for the Datafile type type Datafile struct { @@ -52,6 +54,20 @@ func (_m *Datafile) Name() string { return r0 } +// Open provides a mock function with given fields: +func (_m *Datafile) Open() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + // Read provides a mock function with given fields: func (_m *Datafile) Read() (internal.Entry, int64, error) { ret := _m.Called() diff --git a/internal/mocks/indexer.go b/internal/mocks/indexer.go index 7680aac..519a0d2 100644 --- a/internal/mocks/indexer.go +++ b/internal/mocks/indexer.go @@ -2,9 +2,11 @@ package mocks -import art "github.com/plar/go-adaptive-radix-tree" +import ( + art "github.com/plar/go-adaptive-radix-tree" -import mock "github.com/stretchr/testify/mock" + mock "github.com/stretchr/testify/mock" +) // Indexer is an autogenerated mock type for the Indexer type type Indexer struct {