mirror of
https://github.com/taigrr/bitcask
synced 2025-01-18 04:03:17 -08:00
Add all files again with v2 postfix to URL
This commit is contained in:
110
v2/internal/data/codec/decoder.go
Normal file
110
v2/internal/data/codec/decoder.go
Normal file
@@ -0,0 +1,110 @@
|
||||
package codec
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"git.mills.io/prologic/bitcask/v2/internal"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
var (
|
||||
errInvalidKeyOrValueSize = errors.New("key/value size is invalid")
|
||||
errCantDecodeOnNilEntry = errors.New("can't decode on nil entry")
|
||||
errTruncatedData = errors.New("data is truncated")
|
||||
)
|
||||
|
||||
// NewDecoder creates a streaming Entry decoder.
|
||||
func NewDecoder(r io.Reader, maxKeySize uint32, maxValueSize uint64) *Decoder {
|
||||
return &Decoder{
|
||||
r: r,
|
||||
maxKeySize: maxKeySize,
|
||||
maxValueSize: maxValueSize,
|
||||
}
|
||||
}
|
||||
|
||||
// Decoder wraps an underlying io.Reader and allows you to stream
|
||||
// Entry decodings on it.
|
||||
type Decoder struct {
|
||||
r io.Reader
|
||||
maxKeySize uint32
|
||||
maxValueSize uint64
|
||||
}
|
||||
|
||||
// Decode decodes the next Entry from the current stream
|
||||
func (d *Decoder) Decode(v *internal.Entry) (int64, error) {
|
||||
if v == nil {
|
||||
return 0, errCantDecodeOnNilEntry
|
||||
}
|
||||
|
||||
prefixBuf := make([]byte, keySize+valueSize)
|
||||
|
||||
_, err := io.ReadFull(d.r, prefixBuf)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
actualKeySize, actualValueSize, err := getKeyValueSizes(prefixBuf, d.maxKeySize, d.maxValueSize)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
buf := make([]byte, uint64(actualKeySize)+actualValueSize+checksumSize+ttlSize)
|
||||
if _, err = io.ReadFull(d.r, buf); err != nil {
|
||||
return 0, errTruncatedData
|
||||
}
|
||||
|
||||
decodeWithoutPrefix(buf, actualKeySize, v)
|
||||
return int64(keySize + valueSize + uint64(actualKeySize) + actualValueSize + checksumSize + ttlSize), nil
|
||||
}
|
||||
|
||||
// DecodeEntry decodes a serialized entry
|
||||
func DecodeEntry(b []byte, e *internal.Entry, maxKeySize uint32, maxValueSize uint64) error {
|
||||
valueOffset, _, err := getKeyValueSizes(b, maxKeySize, maxValueSize)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "key/value sizes are invalid")
|
||||
}
|
||||
|
||||
decodeWithoutPrefix(b[keySize+valueSize:], valueOffset, e)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getKeyValueSizes(buf []byte, maxKeySize uint32, maxValueSize uint64) (uint32, uint64, error) {
|
||||
actualKeySize := binary.BigEndian.Uint32(buf[:keySize])
|
||||
actualValueSize := binary.BigEndian.Uint64(buf[keySize:])
|
||||
|
||||
if (maxKeySize > 0 && actualKeySize > maxKeySize) || (maxValueSize > 0 && actualValueSize > maxValueSize) || actualKeySize == 0 {
|
||||
|
||||
return 0, 0, errInvalidKeyOrValueSize
|
||||
}
|
||||
|
||||
return actualKeySize, actualValueSize, nil
|
||||
}
|
||||
|
||||
func decodeWithoutPrefix(buf []byte, valueOffset uint32, v *internal.Entry) {
|
||||
v.Key = buf[:valueOffset]
|
||||
v.Value = buf[valueOffset : len(buf)-checksumSize-ttlSize]
|
||||
v.Checksum = binary.BigEndian.Uint32(buf[len(buf)-checksumSize-ttlSize : len(buf)-ttlSize])
|
||||
v.Expiry = getKeyExpiry(buf)
|
||||
}
|
||||
|
||||
func getKeyExpiry(buf []byte) *time.Time {
|
||||
expiry := binary.BigEndian.Uint64(buf[len(buf)-ttlSize:])
|
||||
if expiry == uint64(0) {
|
||||
return nil
|
||||
}
|
||||
t := time.Unix(int64(expiry), 0).UTC()
|
||||
return &t
|
||||
}
|
||||
|
||||
// IsCorruptedData indicates if the error correspondes to possible data corruption
|
||||
func IsCorruptedData(err error) bool {
|
||||
switch err {
|
||||
case errCantDecodeOnNilEntry, errInvalidKeyOrValueSize, errTruncatedData:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
130
v2/internal/data/codec/decoder_test.go
Normal file
130
v2/internal/data/codec/decoder_test.go
Normal file
@@ -0,0 +1,130 @@
|
||||
package codec
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.mills.io/prologic/bitcask/v2/internal"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestDecodeOnNilEntry(t *testing.T) {
|
||||
t.Parallel()
|
||||
assert := assert.New(t)
|
||||
decoder := NewDecoder(&bytes.Buffer{}, 1, 1)
|
||||
|
||||
_, err := decoder.Decode(nil)
|
||||
if assert.Error(err) {
|
||||
assert.Equal(errCantDecodeOnNilEntry, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShortPrefix(t *testing.T) {
|
||||
t.Parallel()
|
||||
assert := assert.New(t)
|
||||
maxKeySize, maxValueSize := uint32(10), uint64(20)
|
||||
prefix := make([]byte, keySize+valueSize)
|
||||
binary.BigEndian.PutUint32(prefix, 1)
|
||||
binary.BigEndian.PutUint64(prefix[keySize:], 1)
|
||||
|
||||
truncBytesCount := 2
|
||||
buf := bytes.NewBuffer(prefix[:keySize+valueSize-truncBytesCount])
|
||||
decoder := NewDecoder(buf, maxKeySize, maxValueSize)
|
||||
_, err := decoder.Decode(&internal.Entry{})
|
||||
if assert.Error(err) {
|
||||
assert.Equal(io.ErrUnexpectedEOF, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidValueKeySizes(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
maxKeySize, maxValueSize := uint32(10), uint64(20)
|
||||
|
||||
tests := []struct {
|
||||
keySize uint32
|
||||
valueSize uint64
|
||||
name string
|
||||
}{
|
||||
{keySize: 0, valueSize: 5, name: "zero key size"}, //zero value size is correct for tombstones
|
||||
{keySize: 11, valueSize: 5, name: "key size overflow"},
|
||||
{keySize: 5, valueSize: 21, name: "value size overflow"},
|
||||
{keySize: 11, valueSize: 21, name: "key and value size overflow"},
|
||||
}
|
||||
|
||||
for i := range tests {
|
||||
i := i
|
||||
t.Run(tests[i].name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
prefix := make([]byte, keySize+valueSize)
|
||||
binary.BigEndian.PutUint32(prefix, tests[i].keySize)
|
||||
binary.BigEndian.PutUint64(prefix[keySize:], tests[i].valueSize)
|
||||
|
||||
buf := bytes.NewBuffer(prefix)
|
||||
decoder := NewDecoder(buf, maxKeySize, maxValueSize)
|
||||
_, err := decoder.Decode(&internal.Entry{})
|
||||
if assert.Error(err) {
|
||||
assert.Equal(errInvalidKeyOrValueSize, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTruncatedData(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
maxKeySize, maxValueSize := uint32(10), uint64(20)
|
||||
|
||||
key := []byte("foo")
|
||||
value := []byte("bar")
|
||||
data := make([]byte, keySize+valueSize+len(key)+len(value)+checksumSize)
|
||||
|
||||
binary.BigEndian.PutUint32(data, uint32(len(key)))
|
||||
binary.BigEndian.PutUint64(data[keySize:], uint64(len(value)))
|
||||
copy(data[keySize+valueSize:], key)
|
||||
copy(data[keySize+valueSize+len(key):], value)
|
||||
copy(data[keySize+valueSize+len(key)+len(value):], bytes.Repeat([]byte("0"), checksumSize))
|
||||
|
||||
tests := []struct {
|
||||
data []byte
|
||||
name string
|
||||
}{
|
||||
{data: data[:keySize+valueSize+len(key)-1], name: "truncated key"},
|
||||
{data: data[:keySize+valueSize+len(key)+len(value)-1], name: "truncated value"},
|
||||
{data: data[:keySize+valueSize+len(key)+len(value)+checksumSize-1], name: "truncated checksum"},
|
||||
}
|
||||
|
||||
for i := range tests {
|
||||
i := i
|
||||
t.Run(tests[i].name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
buf := bytes.NewBuffer(tests[i].data)
|
||||
decoder := NewDecoder(buf, maxKeySize, maxValueSize)
|
||||
_, err := decoder.Decode(&internal.Entry{})
|
||||
if assert.Error(err) {
|
||||
assert.Equal(errTruncatedData, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeWithoutPrefix(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
e := internal.Entry{}
|
||||
buf := []byte{0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 7, 109, 121, 107, 101, 121, 109, 121, 118, 97, 108, 117, 101, 0, 6, 81, 189, 0, 0, 0, 0, 95, 117, 28, 0}
|
||||
valueOffset := uint32(5)
|
||||
mockTime := time.Date(2020, 10, 1, 0, 0, 0, 0, time.UTC)
|
||||
expectedEntry := internal.Entry{
|
||||
Key: []byte("mykey"),
|
||||
Value: []byte("myvalue"),
|
||||
Checksum: 414141,
|
||||
Expiry: &mockTime,
|
||||
}
|
||||
decodeWithoutPrefix(buf[keySize+valueSize:], valueOffset, &e)
|
||||
assert.Equal(expectedEntry.Key, e.Key)
|
||||
assert.Equal(expectedEntry.Value, e.Value)
|
||||
assert.Equal(expectedEntry.Checksum, e.Checksum)
|
||||
assert.Equal(expectedEntry.Offset, e.Offset)
|
||||
assert.Equal(*expectedEntry.Expiry, *e.Expiry)
|
||||
}
|
||||
69
v2/internal/data/codec/encoder.go
Normal file
69
v2/internal/data/codec/encoder.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package codec
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
|
||||
"git.mills.io/prologic/bitcask/v2/internal"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
keySize = 4
|
||||
valueSize = 8
|
||||
checksumSize = 4
|
||||
ttlSize = 8
|
||||
MetaInfoSize = keySize + valueSize + checksumSize + ttlSize
|
||||
)
|
||||
|
||||
// NewEncoder creates a streaming Entry encoder.
|
||||
func NewEncoder(w io.Writer) *Encoder {
|
||||
return &Encoder{w: bufio.NewWriter(w)}
|
||||
}
|
||||
|
||||
// Encoder wraps an underlying io.Writer and allows you to stream
|
||||
// Entry encodings on it.
|
||||
type Encoder struct {
|
||||
w *bufio.Writer
|
||||
}
|
||||
|
||||
// Encode takes any Entry and streams it to the underlying writer.
|
||||
// Messages are framed with a key-length and value-length prefix.
|
||||
func (e *Encoder) Encode(msg internal.Entry) (int64, error) {
|
||||
var bufKeyValue = make([]byte, keySize+valueSize)
|
||||
binary.BigEndian.PutUint32(bufKeyValue[:keySize], uint32(len(msg.Key)))
|
||||
binary.BigEndian.PutUint64(bufKeyValue[keySize:keySize+valueSize], uint64(len(msg.Value)))
|
||||
if _, err := e.w.Write(bufKeyValue); err != nil {
|
||||
return 0, errors.Wrap(err, "failed writing key & value length prefix")
|
||||
}
|
||||
|
||||
if _, err := e.w.Write(msg.Key); err != nil {
|
||||
return 0, errors.Wrap(err, "failed writing key data")
|
||||
}
|
||||
if _, err := e.w.Write(msg.Value); err != nil {
|
||||
return 0, errors.Wrap(err, "failed writing value data")
|
||||
}
|
||||
|
||||
bufChecksumSize := bufKeyValue[:checksumSize]
|
||||
binary.BigEndian.PutUint32(bufChecksumSize, msg.Checksum)
|
||||
if _, err := e.w.Write(bufChecksumSize); err != nil {
|
||||
return 0, errors.Wrap(err, "failed writing checksum data")
|
||||
}
|
||||
|
||||
bufTTL := bufKeyValue[:ttlSize]
|
||||
if msg.Expiry == nil {
|
||||
binary.BigEndian.PutUint64(bufTTL, uint64(0))
|
||||
} else {
|
||||
binary.BigEndian.PutUint64(bufTTL, uint64(msg.Expiry.Unix()))
|
||||
}
|
||||
if _, err := e.w.Write(bufTTL); err != nil {
|
||||
return 0, errors.Wrap(err, "failed writing ttl data")
|
||||
}
|
||||
|
||||
if err := e.w.Flush(); err != nil {
|
||||
return 0, errors.Wrap(err, "failed flushing data")
|
||||
}
|
||||
|
||||
return int64(keySize + valueSize + len(msg.Key) + len(msg.Value) + checksumSize + ttlSize), nil
|
||||
}
|
||||
32
v2/internal/data/codec/encoder_test.go
Normal file
32
v2/internal/data/codec/encoder_test.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package codec
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.mills.io/prologic/bitcask/v2/internal"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestEncode(t *testing.T) {
|
||||
t.Parallel()
|
||||
assert := assert.New(t)
|
||||
|
||||
var buf bytes.Buffer
|
||||
mockTime := time.Date(2020, 10, 1, 0, 0, 0, 0, time.UTC)
|
||||
encoder := NewEncoder(&buf)
|
||||
_, err := encoder.Encode(internal.Entry{
|
||||
Key: []byte("mykey"),
|
||||
Value: []byte("myvalue"),
|
||||
Checksum: 414141,
|
||||
Offset: 424242,
|
||||
Expiry: &mockTime,
|
||||
})
|
||||
|
||||
expectedHex := "0000000500000000000000076d796b65796d7976616c7565000651bd000000005f751c00"
|
||||
if assert.NoError(err) {
|
||||
assert.Equal(expectedHex, hex.EncodeToString(buf.Bytes()))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user