taigrr/bitcask
1
0
Fork 0
mirror of https://github.com/taigrr/bitcask synced 2025-01-18 04:03:17 -08:00
Code Issues Projects Releases Wiki Activity

Makefile setup & key/value coherent datatypes & refactoring (#98)

Browse Source 
* internal/data: comment exported functions

* internal/data: make smaller codec exported api surface

* make key and value sizes serializing bubble up to everything

* Makefile setup & go mod tidy
This commit is contained in:
Ignacio Hagopian
2019-09-12 10:44:26 -03:00
committed by GitHub
parent 7e0fa151f7
commit 5be114adab
14 changed files with 112 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/config/config.go
View File

@@ -8,10 +8,10 @@ import (
// Config contains the bitcask configuration parameters
type Config struct {
MaxDatafileSize int `json:"max_datafile_size"`
MaxKeySize int `json:"max_key_size"`
MaxValueSize int `json:"max_value_size"`
Sync bool `json:"sync"`
MaxDatafileSize int `json:"max_datafile_size"`
MaxKeySize uint32 `json:"max_key_size"`
MaxValueSize uint64 `json:"max_value_size"`
Sync bool `json:"sync"`
}
// Load loads a configuration from the given path

71
internal/data/codec.go
View File

@@ -10,11 +10,17 @@ import (
)
const (
KeySize = 4
ValueSize = 8
keySize = 4
valueSize = 8
checksumSize = 4
)
var (
// ErrInvalidKeyOrValueSize indicates a serialized key/value size
// which is greater than specified limit
ErrInvalidKeyOrValueSize = errors.New("key/value size is invalid")
)
// NewEncoder creates a streaming Entry encoder.
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{w: bufio.NewWriter(w)}
@@ -29,9 +35,9 @@ type Encoder struct {
// Encode takes any Entry and streams it to the underlying writer.
// Messages are framed with a key-length and value-length prefix.
func (e *Encoder) Encode(msg internal.Entry) (int64, error) {
var bufKeyValue = make([]byte, KeySize+ValueSize)
binary.BigEndian.PutUint32(bufKeyValue[:KeySize], uint32(len(msg.Key)))
binary.BigEndian.PutUint64(bufKeyValue[KeySize:KeySize+ValueSize], uint64(len(msg.Value)))
var bufKeyValue = make([]byte, keySize+valueSize)
binary.BigEndian.PutUint32(bufKeyValue[:keySize], uint32(len(msg.Key)))
binary.BigEndian.PutUint64(bufKeyValue[keySize:keySize+valueSize], uint64(len(msg.Value)))
if _, err := e.w.Write(bufKeyValue); err != nil {
return 0, errors.Wrap(err, "failed writing key & value length prefix")
}
@@ -53,46 +59,73 @@ func (e *Encoder) Encode(msg internal.Entry) (int64, error) {
return 0, errors.Wrap(err, "failed flushing data")
}
return int64(KeySize + ValueSize + len(msg.Key) + len(msg.Value) + checksumSize), nil
return int64(keySize + valueSize + len(msg.Key) + len(msg.Value) + checksumSize), nil
}
// NewDecoder creates a streaming Entry decoder.
func NewDecoder(r io.Reader) *Decoder {
return &Decoder{r: r}
func NewDecoder(r io.Reader, maxKeySize uint32, maxValueSize uint64) *Decoder {
return &Decoder{
r: r,
maxKeySize: maxKeySize,
maxValueSize: maxValueSize,
}
}
// Decoder wraps an underlying io.Reader and allows you to stream
// Entry decodings on it.
type Decoder struct {
r io.Reader
r io.Reader
maxKeySize uint32
maxValueSize uint64
}
// Decode decodes the next Entry from the current stream
func (d *Decoder) Decode(v *internal.Entry) (int64, error) {
prefixBuf := make([]byte, KeySize+ValueSize)
prefixBuf := make([]byte, keySize+valueSize)
_, err := io.ReadFull(d.r, prefixBuf)
if err != nil {
return 0, err
}
actualKeySize, actualValueSize := GetKeyValueSizes(prefixBuf)
buf := make([]byte, actualKeySize+actualValueSize+checksumSize)
actualKeySize, actualValueSize, err := getKeyValueSizes(prefixBuf, d.maxKeySize, d.maxValueSize)
if err != nil {
return 0, errors.Wrap(err, "error while getting key/value serialized sizes")
}
buf := make([]byte, uint64(actualKeySize)+actualValueSize+checksumSize)
if _, err = io.ReadFull(d.r, buf); err != nil {
return 0, errors.Wrap(translateError(err), "failed reading saved data")
}
DecodeWithoutPrefix(buf, actualKeySize, v)
return int64(KeySize + ValueSize + actualKeySize + actualValueSize + checksumSize), nil
decodeWithoutPrefix(buf, actualKeySize, v)
return int64(keySize + valueSize + uint64(actualKeySize) + actualValueSize + checksumSize), nil
}
func GetKeyValueSizes(buf []byte) (uint64, uint64) {
actualKeySize := binary.BigEndian.Uint32(buf[:KeySize])
actualValueSize := binary.BigEndian.Uint64(buf[KeySize:])
// DecodeEntry decodes a serialized entry
func DecodeEntry(b []byte, e *internal.Entry, maxKeySize uint32, maxValueSize uint64) error {
valueOffset, _, err := getKeyValueSizes(b, maxKeySize, maxValueSize)
if err != nil {
return errors.Wrap(err, "key/value sizes are invalid")
}
return uint64(actualKeySize), actualValueSize
decodeWithoutPrefix(b[keySize+valueSize:], valueOffset, e)
return nil
}
func DecodeWithoutPrefix(buf []byte, valueOffset uint64, v *internal.Entry) {
func getKeyValueSizes(buf []byte, maxKeySize uint32, maxValueSize uint64) (uint32, uint64, error) {
actualKeySize := binary.BigEndian.Uint32(buf[:keySize])
actualValueSize := binary.BigEndian.Uint64(buf[keySize:])
if actualKeySize > maxKeySize || actualValueSize > maxValueSize {
return 0, 0, ErrInvalidKeyOrValueSize
}
return actualKeySize, actualValueSize, nil
}
func decodeWithoutPrefix(buf []byte, valueOffset uint32, v *internal.Entry) {
v.Key = buf[:valueOffset]
v.Value = buf[valueOffset : len(buf)-checksumSize]
v.Checksum = binary.BigEndian.Uint32(buf[len(buf)-checksumSize:])

42
internal/data/datafile.go
View File

@@ -36,16 +36,19 @@ type Datafile interface {
type datafile struct {
sync.RWMutex
id int
r *os.File
ra *mmap.ReaderAt
w *os.File
offset int64
dec *Decoder
enc *Encoder
id int
r *os.File
ra *mmap.ReaderAt
w *os.File
offset int64
dec *Decoder
enc *Encoder
maxKeySize uint32
maxValueSize uint64
}
func NewDatafile(path string, id int, readonly bool) (Datafile, error) {
// NewDatafile opens an existing datafile
func NewDatafile(path string, id int, readonly bool, maxKeySize uint32, maxValueSize uint64) (Datafile, error) {
var (
r *os.File
ra *mmap.ReaderAt
@@ -78,17 +81,19 @@ func NewDatafile(path string, id int, readonly bool) (Datafile, error) {
offset := stat.Size()
dec := NewDecoder(r)
dec := NewDecoder(r, maxKeySize, maxValueSize)
enc := NewEncoder(w)
return &datafile{
id: id,
r: r,
ra: ra,
w: w,
offset: offset,
dec: dec,
enc: enc,
id: id,
r: r,
ra: ra,
w: w,
offset: offset,
dec: dec,
enc: enc,
maxKeySize: maxKeySize,
maxValueSize: maxValueSize,
}, nil
}
@@ -131,6 +136,7 @@ func (df *datafile) Size() int64 {
return df.offset
}
// Read reads the next entry from the datafile
func (df *datafile) Read() (e internal.Entry, n int64, err error) {
df.Lock()
defer df.Unlock()
@@ -143,6 +149,7 @@ func (df *datafile) Read() (e internal.Entry, n int64, err error) {
return
}
// ReadAt the entry located at index offset with expected serialized size
func (df *datafile) ReadAt(index, size int64) (e internal.Entry, err error) {
var n int
@@ -161,8 +168,7 @@ func (df *datafile) ReadAt(index, size int64) (e internal.Entry, err error) {
return
}
valueOffset, _ := GetKeyValueSizes(b)
DecodeWithoutPrefix(b[KeySize+ValueSize:], valueOffset, &e)
DecodeEntry(b, &e, df.maxKeySize, df.maxValueSize)
return
}

4
internal/index/codec_index.go
View File

@@ -24,7 +24,7 @@ const (
sizeSize = int64Size
)
func readKeyBytes(r io.Reader, maxKeySize int) ([]byte, error) {
func readKeyBytes(r io.Reader, maxKeySize uint32) ([]byte, error) {
s := make([]byte, int32Size)
_, err := io.ReadFull(r, s)
if err != nil {
@@ -87,7 +87,7 @@ func writeItem(item internal.Item, w io.Writer) error {
}
// ReadIndex reads a persisted from a io.Reader into a Tree
func readIndex(r io.Reader, t art.Tree, maxKeySize int) error {
func readIndex(r io.Reader, t art.Tree, maxKeySize uint32) error {
for {
key, err := readKeyBytes(r, maxKeySize)
if err != nil {

2
internal/index/codec_index_test.go
View File

@@ -94,7 +94,7 @@ func TestReadCorruptedData(t *testing.T) {
table := []struct {
name string
err error
maxKeySize int
maxKeySize uint32
data []byte
}{
{name: "key-data-overflow", err: errKeySizeTooLarge, maxKeySize: 1024, data: overflowKeySize},

4
internal/index/index.go
View File

@@ -8,7 +8,7 @@ import (
)
type Indexer interface {
Load(path string, maxkeySize int) (art.Tree, bool, error)
Load(path string, maxkeySize uint32) (art.Tree, bool, error)
Save(t art.Tree, path string) error
}
@@ -18,7 +18,7 @@ func NewIndexer() Indexer {
type indexer struct{}
func (i *indexer) Load(path string, maxKeySize int) (art.Tree, bool, error) {
func (i *indexer) Load(path string, maxKeySize uint32) (art.Tree, bool, error) {
t := art.New()
if !internal.Exists(path) {

8
internal/mocks/indexer.go
View File

@@ -12,11 +12,11 @@ type Indexer struct {
}
// Load provides a mock function with given fields: path, maxkeySize
func (_m *Indexer) Load(path string, maxkeySize int) (art.Tree, bool, error) {
func (_m *Indexer) Load(path string, maxkeySize uint32) (art.Tree, bool, error) {
ret := _m.Called(path, maxkeySize)
var r0 art.Tree
if rf, ok := ret.Get(0).(func(string, int) art.Tree); ok {
if rf, ok := ret.Get(0).(func(string, uint32) art.Tree); ok {
r0 = rf(path, maxkeySize)
} else {
if ret.Get(0) != nil {
@@ -25,14 +25,14 @@ func (_m *Indexer) Load(path string, maxkeySize int) (art.Tree, bool, error) {
}
var r1 bool
if rf, ok := ret.Get(1).(func(string, int) bool); ok {
if rf, ok := ret.Get(1).(func(string, uint32) bool); ok {
r1 = rf(path, maxkeySize)
} else {
r1 = ret.Get(1).(bool)
}
var r2 error
if rf, ok := ret.Get(2).(func(string, int) error); ok {
if rf, ok := ret.Get(2).(func(string, uint32) error); ok {
r2 = rf(path, maxkeySize)
} else {
r2 = ret.Error(2)