taigrr/bitcask
1
0
Fork 0
mirror of https://github.com/taigrr/bitcask synced 2025-01-18 04:03:17 -08:00
Code Issues Projects Releases Wiki Activity

Refactored some internals and removed timestamp field (unsure why it was needed in the original paper)

Browse Source
This commit is contained in:
James Mills 2019-03-16 12:40:24 +10:00
parent cb00b11dd7
commit 67ab944db7
No known key found for this signature in database
GPG Key ID: AC4C014F1440EBD6
6 changed files with 40 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
bitcask.go
View File

@ -8,7 +8,6 @@ import (
"os"
"path/filepath"
"strings"
"time"
"github.com/gofrs/flock"
"github.com/prologic/trie"
@ -57,14 +56,14 @@ func (b *Bitcask) Get(key string) ([]byte, error) {
df = b.datafiles[item.FileID]
}
e, err := df.ReadAt(item.Index)
e, err := df.ReadAt(item.Offset)
if err != nil {
return nil, err
}
crc := crc32.ChecksumIEEE(e.Value)
if crc != e.CRC {
return nil, fmt.Errorf("error: crc checksum falied %s %d != %d", key, e.CRC, crc)
checksum := crc32.ChecksumIEEE(e.Value)
if checksum != e.Checksum {
return nil, fmt.Errorf("error: checksum falied %s %d != %d", key, e.Checksum, checksum)
}
return e.Value, nil
@ -78,12 +77,12 @@ func (b *Bitcask) Put(key string, value []byte) error {
return fmt.Errorf("error: value too large %d > %d", len(value), b.opts.MaxValueSize)
}
index, err := b.put(key, value)
offset, err := b.put(key, value)
if err != nil {
return err
}
item := b.keydir.Add(key, b.curr.id, index, time.Now().Unix())
item := b.keydir.Add(key, b.curr.id, offset)
b.trie.Add(key, item)
return nil
@ -211,7 +210,7 @@ func Merge(path string, force bool) error {
continue
}
keydir.Add(e.Key, ids[i], e.Index, e.Timestamp)
keydir.Add(e.Key, ids[i], e.Offset)
}
tempdf, err := NewDatafile(temp, id, false)
@ -222,7 +221,7 @@ func Merge(path string, force bool) error {
for key := range keydir.Keys() {
item, _ := keydir.Get(key)
e, err := df.ReadAt(item.Index)
e, err := df.ReadAt(item.Offset)
if err != nil {
return err
}
@ -304,7 +303,7 @@ func Open(path string, options ...func(*Bitcask) error) (*Bitcask, error) {
for key := range hint.Keys() {
item, _ := hint.Get(key)
_ = keydir.Add(key, item.FileID, item.Index, item.Timestamp)
_ = keydir.Add(key, item.FileID, item.Offset)
trie.Add(key, item)
}
} else {
@ -323,7 +322,7 @@ func Open(path string, options ...func(*Bitcask) error) (*Bitcask, error) {
continue
}
item := keydir.Add(e.Key, ids[i], e.Index, e.Timestamp)
item := keydir.Add(e.Key, ids[i], e.Offset)
trie.Add(e.Key, item)
}
}

6
datafile.go
View File

@ -5,7 +5,6 @@ import (
"os"
"path/filepath"
"sync"
"time"
"github.com/pkg/errors"
@ -128,8 +127,7 @@ func (df *Datafile) Write(e pb.Entry) (int64, error) {
df.Lock()
defer df.Unlock()
e.Index = df.offset
e.Timestamp = time.Now().Unix()
e.Offset = df.offset
n, err := df.enc.Encode(&e)
if err != nil {
@ -137,5 +135,5 @@ func (df *Datafile) Write(e pb.Entry) (int64, error) {
}
df.offset += n
return e.Index, nil
return e.Offset, nil
}

8
entry.go
View File

@ -7,11 +7,11 @@ import (
)
func NewEntry(key string, value []byte) pb.Entry {
crc := crc32.ChecksumIEEE(value)
checksum := crc32.ChecksumIEEE(value)
return pb.Entry{
CRC: crc,
Key: key,
Value: value,
Checksum: checksum,
Key: key,
Value: value,
}
}

12
keydir.go
View File

@ -9,9 +9,8 @@ import (
)
type Item struct {
FileID int
Index int64
Timestamp int64
FileID int
Offset int64
}
type Keydir struct {
@ -25,11 +24,10 @@ func NewKeydir() *Keydir {
}
}
func (k *Keydir) Add(key string, fileid int, index, timestamp int64) Item {
func (k *Keydir) Add(key string, fileid int, offset int64) Item {
item := Item{
FileID: fileid,
Index: index,
Timestamp: timestamp,
FileID: fileid,
Offset: offset,
}
k.Lock()

43
proto/entry.pb.go
View File

@ -19,11 +19,10 @@ var _ = math.Inf
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
type Entry struct {
CRC uint32 `protobuf:"varint,1,opt,name=CRC,proto3" json:"CRC,omitempty"`
Checksum uint32 `protobuf:"varint,1,opt,name=Checksum,proto3" json:"Checksum,omitempty"`
Key string `protobuf:"bytes,2,opt,name=Key,proto3" json:"Key,omitempty"`
Index int64 `protobuf:"varint,3,opt,name=Index,proto3" json:"Index,omitempty"`
Offset int64 `protobuf:"varint,3,opt,name=Offset,proto3" json:"Offset,omitempty"`
Value []byte `protobuf:"bytes,4,opt,name=Value,proto3" json:"Value,omitempty"`
Timestamp int64 `protobuf:"varint,5,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -33,7 +32,7 @@ func (m *Entry) Reset() { *m = Entry{} }
func (m *Entry) String() string { return proto.CompactTextString(m) }
func (*Entry) ProtoMessage() {}
func (*Entry) Descriptor() ([]byte, []int) {
return fileDescriptor_entry_4f5906245d08394f, []int{0}
return fileDescriptor_entry_3e91842c99935ae2, []int{0}
}
func (m *Entry) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Entry.Unmarshal(m, b)
@ -53,9 +52,9 @@ func (m *Entry) XXX_DiscardUnknown() {
var xxx_messageInfo_Entry proto.InternalMessageInfo
func (m *Entry) GetCRC() uint32 {
func (m *Entry) GetChecksum() uint32 {
if m != nil {
return m.CRC
return m.Checksum
}
return 0
}
@ -67,9 +66,9 @@ func (m *Entry) GetKey() string {
return ""
}
func (m *Entry) GetIndex() int64 {
func (m *Entry) GetOffset() int64 {
if m != nil {
return m.Index
return m.Offset
}
return 0
}
@ -81,28 +80,20 @@ func (m *Entry) GetValue() []byte {
return nil
}
func (m *Entry) GetTimestamp() int64 {
if m != nil {
return m.Timestamp
}
return 0
}
func init() {
proto.RegisterType((*Entry)(nil), "proto.Entry")
}
func init() { proto.RegisterFile("entry.proto", fileDescriptor_entry_4f5906245d08394f) }
func init() { proto.RegisterFile("entry.proto", fileDescriptor_entry_3e91842c99935ae2) }
var fileDescriptor_entry_4f5906245d08394f = []byte{
// 134 bytes of a gzipped FileDescriptorProto
var fileDescriptor_entry_3e91842c99935ae2 = []byte{
// 126 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4e, 0xcd, 0x2b, 0x29,
0xaa, 0xd4, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x05, 0x53, 0x4a, 0xa5, 0x5c, 0xac, 0xae,
0x20, 0x51, 0x21, 0x01, 0x2e, 0x66, 0xe7, 0x20, 0x67, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xde, 0x20,
0x10, 0x13, 0x24, 0xe2, 0x9d, 0x5a, 0x29, 0xc1, 0xa4, 0xc0, 0xa8, 0xc1, 0x19, 0x04, 0x62, 0x0a,
0x89, 0x70, 0xb1, 0x7a, 0xe6, 0xa5, 0xa4, 0x56, 0x48, 0x30, 0x2b, 0x30, 0x6a, 0x30, 0x07, 0x41,
0x38, 0x20, 0xd1, 0xb0, 0xc4, 0x9c, 0xd2, 0x54, 0x09, 0x16, 0x05, 0x46, 0x0d, 0x9e, 0x20, 0x08,
0x47, 0x48, 0x86, 0x8b, 0x33, 0x24, 0x33, 0x37, 0xb5, 0xb8, 0x24, 0x31, 0xb7, 0x40, 0x82, 0x15,
0xac, 0x1e, 0x21, 0x90, 0xc4, 0x06, 0xb6, 0xdd, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0x07, 0x99,
0x47, 0xb9, 0x93, 0x00, 0x00, 0x00,
0xaa, 0xd4, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x05, 0x53, 0x4a, 0xc9, 0x5c, 0xac, 0xae,
0x20, 0x51, 0x21, 0x29, 0x2e, 0x0e, 0xe7, 0x8c, 0xd4, 0xe4, 0xec, 0xe2, 0xd2, 0x5c, 0x09, 0x46,
0x05, 0x46, 0x0d, 0xde, 0x20, 0x38, 0x5f, 0x48, 0x80, 0x8b, 0xd9, 0x3b, 0xb5, 0x52, 0x82, 0x49,
0x81, 0x51, 0x83, 0x33, 0x08, 0xc4, 0x14, 0x12, 0xe3, 0x62, 0xf3, 0x4f, 0x4b, 0x2b, 0x4e, 0x2d,
0x91, 0x60, 0x56, 0x60, 0xd4, 0x60, 0x0e, 0x82, 0xf2, 0x84, 0x44, 0xb8, 0x58, 0xc3, 0x12, 0x73,
0x4a, 0x53, 0x25, 0x58, 0x14, 0x18, 0x35, 0x78, 0x82, 0x20, 0x9c, 0x24, 0x36, 0xb0, 0x5d, 0xc6,
0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x76, 0xd2, 0x3e, 0x83, 0x81, 0x00, 0x00, 0x00,
}

5
proto/entry.proto
View File

@ -3,9 +3,8 @@ syntax = "proto3";
package proto;
message Entry {
uint32 CRC = 1;
uint32 Checksum = 1;
string Key = 2;
int64 Index = 3;
int64 Offset = 3;
bytes Value = 4;
int64 Timestamp = 5;
}