taigrr/bitcask
1
0
Fork 0
mirror of https://github.com/taigrr/bitcask synced 2025-01-18 04:03:17 -08:00
Code Issues Projects Releases Wiki Activity

Makefile setup & key/value coherent datatypes & refactoring (#98)

Browse Source 
* internal/data: comment exported functions

* internal/data: make smaller codec exported api surface

* make key and value sizes serializing bubble up to everything

* Makefile setup & go mod tidy
This commit is contained in:
Ignacio Hagopian
2019-09-12 10:44:26 -03:00
committed by GitHub
parent 7e0fa151f7
commit 5be114adab
14 changed files with 112 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
internal/data/codec.go
View File

@@ -10,11 +10,17 @@ import (
)
const (
KeySize = 4
ValueSize = 8
keySize = 4
valueSize = 8
checksumSize = 4
)
var (
// ErrInvalidKeyOrValueSize indicates a serialized key/value size
// which is greater than specified limit
ErrInvalidKeyOrValueSize = errors.New("key/value size is invalid")
)
// NewEncoder creates a streaming Entry encoder.
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{w: bufio.NewWriter(w)}
@@ -29,9 +35,9 @@ type Encoder struct {
// Encode takes any Entry and streams it to the underlying writer.
// Messages are framed with a key-length and value-length prefix.
func (e *Encoder) Encode(msg internal.Entry) (int64, error) {
var bufKeyValue = make([]byte, KeySize+ValueSize)
binary.BigEndian.PutUint32(bufKeyValue[:KeySize], uint32(len(msg.Key)))
binary.BigEndian.PutUint64(bufKeyValue[KeySize:KeySize+ValueSize], uint64(len(msg.Value)))
var bufKeyValue = make([]byte, keySize+valueSize)
binary.BigEndian.PutUint32(bufKeyValue[:keySize], uint32(len(msg.Key)))
binary.BigEndian.PutUint64(bufKeyValue[keySize:keySize+valueSize], uint64(len(msg.Value)))
if _, err := e.w.Write(bufKeyValue); err != nil {
return 0, errors.Wrap(err, "failed writing key & value length prefix")
}
@@ -53,46 +59,73 @@ func (e *Encoder) Encode(msg internal.Entry) (int64, error) {
return 0, errors.Wrap(err, "failed flushing data")
}
return int64(KeySize + ValueSize + len(msg.Key) + len(msg.Value) + checksumSize), nil
return int64(keySize + valueSize + len(msg.Key) + len(msg.Value) + checksumSize), nil
}
// NewDecoder creates a streaming Entry decoder.
func NewDecoder(r io.Reader) *Decoder {
return &Decoder{r: r}
func NewDecoder(r io.Reader, maxKeySize uint32, maxValueSize uint64) *Decoder {
return &Decoder{
r: r,
maxKeySize: maxKeySize,
maxValueSize: maxValueSize,
}
}
// Decoder wraps an underlying io.Reader and allows you to stream
// Entry decodings on it.
type Decoder struct {
r io.Reader
r io.Reader
maxKeySize uint32
maxValueSize uint64
}
// Decode decodes the next Entry from the current stream
func (d *Decoder) Decode(v *internal.Entry) (int64, error) {
prefixBuf := make([]byte, KeySize+ValueSize)
prefixBuf := make([]byte, keySize+valueSize)
_, err := io.ReadFull(d.r, prefixBuf)
if err != nil {
return 0, err
}
actualKeySize, actualValueSize := GetKeyValueSizes(prefixBuf)
buf := make([]byte, actualKeySize+actualValueSize+checksumSize)
actualKeySize, actualValueSize, err := getKeyValueSizes(prefixBuf, d.maxKeySize, d.maxValueSize)
if err != nil {
return 0, errors.Wrap(err, "error while getting key/value serialized sizes")
}
buf := make([]byte, uint64(actualKeySize)+actualValueSize+checksumSize)
if _, err = io.ReadFull(d.r, buf); err != nil {
return 0, errors.Wrap(translateError(err), "failed reading saved data")
}
DecodeWithoutPrefix(buf, actualKeySize, v)
return int64(KeySize + ValueSize + actualKeySize + actualValueSize + checksumSize), nil
decodeWithoutPrefix(buf, actualKeySize, v)
return int64(keySize + valueSize + uint64(actualKeySize) + actualValueSize + checksumSize), nil
}
func GetKeyValueSizes(buf []byte) (uint64, uint64) {
actualKeySize := binary.BigEndian.Uint32(buf[:KeySize])
actualValueSize := binary.BigEndian.Uint64(buf[KeySize:])
// DecodeEntry decodes a serialized entry
func DecodeEntry(b []byte, e *internal.Entry, maxKeySize uint32, maxValueSize uint64) error {
valueOffset, _, err := getKeyValueSizes(b, maxKeySize, maxValueSize)
if err != nil {
return errors.Wrap(err, "key/value sizes are invalid")
}
return uint64(actualKeySize), actualValueSize
decodeWithoutPrefix(b[keySize+valueSize:], valueOffset, e)
return nil
}
func DecodeWithoutPrefix(buf []byte, valueOffset uint64, v *internal.Entry) {
func getKeyValueSizes(buf []byte, maxKeySize uint32, maxValueSize uint64) (uint32, uint64, error) {
actualKeySize := binary.BigEndian.Uint32(buf[:keySize])
actualValueSize := binary.BigEndian.Uint64(buf[keySize:])
if actualKeySize > maxKeySize || actualValueSize > maxValueSize {
return 0, 0, ErrInvalidKeyOrValueSize
}
return actualKeySize, actualValueSize, nil
}
func decodeWithoutPrefix(buf []byte, valueOffset uint32, v *internal.Entry) {
v.Key = buf[:valueOffset]
v.Value = buf[valueOffset : len(buf)-checksumSize]
v.Checksum = binary.BigEndian.Uint32(buf[len(buf)-checksumSize:])

42
internal/data/datafile.go
View File

@@ -36,16 +36,19 @@ type Datafile interface {
type datafile struct {
sync.RWMutex
id int
r *os.File
ra *mmap.ReaderAt
w *os.File
offset int64
dec *Decoder
enc *Encoder
id int
r *os.File
ra *mmap.ReaderAt
w *os.File
offset int64
dec *Decoder
enc *Encoder
maxKeySize uint32
maxValueSize uint64
}
func NewDatafile(path string, id int, readonly bool) (Datafile, error) {
// NewDatafile opens an existing datafile
func NewDatafile(path string, id int, readonly bool, maxKeySize uint32, maxValueSize uint64) (Datafile, error) {
var (
r *os.File
ra *mmap.ReaderAt
@@ -78,17 +81,19 @@ func NewDatafile(path string, id int, readonly bool) (Datafile, error) {
offset := stat.Size()
dec := NewDecoder(r)
dec := NewDecoder(r, maxKeySize, maxValueSize)
enc := NewEncoder(w)
return &datafile{
id: id,
r: r,
ra: ra,
w: w,
offset: offset,
dec: dec,
enc: enc,
id: id,
r: r,
ra: ra,
w: w,
offset: offset,
dec: dec,
enc: enc,
maxKeySize: maxKeySize,
maxValueSize: maxValueSize,
}, nil
}
@@ -131,6 +136,7 @@ func (df *datafile) Size() int64 {
return df.offset
}
// Read reads the next entry from the datafile
func (df *datafile) Read() (e internal.Entry, n int64, err error) {
df.Lock()
defer df.Unlock()
@@ -143,6 +149,7 @@ func (df *datafile) Read() (e internal.Entry, n int64, err error) {
return
}
// ReadAt the entry located at index offset with expected serialized size
func (df *datafile) ReadAt(index, size int64) (e internal.Entry, err error) {
var n int
@@ -161,8 +168,7 @@ func (df *datafile) ReadAt(index, size int64) (e internal.Entry, err error) {
return
}
valueOffset, _ := GetKeyValueSizes(b)
DecodeWithoutPrefix(b[KeySize+ValueSize:], valueOffset, &e)
DecodeEntry(b, &e, df.maxKeySize, df.maxValueSize)
return
}