Fix install target

Closes #244.

Reviewed-on: https://git.mills.io/prologic/bitcask/pulls/245
Reviewed-by: James Mills <james@mills.io>
Co-authored-by: Tai Groot <tai@taigrr.com>
Co-committed-by: Tai Groot <tai@taigrr.com>

.gitignore

@@ -5,6 +5,7 @@
 
 /tmp
 /dist
+/cacheDb
 /coverage.txt
 
 /bitcask

AUTHORS

@@ -17,3 +17,5 @@ Yash Chandra <yashschandra@gmail.com>
 Yury Fedorov orlangure
 o2gy84 <o2gy84@gmail.com>
 garsue <labs.garsue@gmail.com>
+biozz <ielfimov@gmail.com>
+jason3gb <jason3gb@gmail.com>

CHANGELOG.md

@@ -1,6 +1,33 @@
 
+<a name="v1.0.2"></a>
+## [v1.0.2](https://git.mills.io/prologic/bitcask/compare/v1.0.1...v1.0.2) (2021-11-01)
+
+### Bug Fixes
+
+* Fix a data race in Datafile.ReadAt()
+* Fix release tool
+
+
+<a name="v1.0.1"></a>
+## [v1.0.1](https://git.mills.io/prologic/bitcask/compare/v1.0.0...v1.0.1) (2021-10-31)
+
+### Features
+
+* Add ErrBadConfig and ErrBadMetadata as errors that consumers can check and use (#241)
+* Add key prefix matching to KEYS command (#237)
+
+### Updates
+
+* Update CHANGELOG for v1.0.1
+* Update image target
+
+
 <a name="v1.0.0"></a>
-## [v1.0.0](https://git.mills.io/prologic/bitcask/compare/1.0.0...v1.0.0) (0001-01-01)
+## [v1.0.0](https://git.mills.io/prologic/bitcask/compare/1.0.0...v1.0.0) (2021-07-24)
+
+### Updates
+
+* Update CHANGELOG for v1.0.0
 
 
 <a name="1.0.0"></a>

Makefile

@@ -3,6 +3,10 @@
 CGO_ENABLED=0
 VERSION=$(shell git describe --abbrev=0 --tags)
 COMMIT=$(shell git rev-parse --short HEAD)
+BUILD=$(shell git show -s --pretty=format:%cI)
+GOCMD=go
+
+DESTDIR=/usr/local/bin
 
 all: dev
 
@@ -11,46 +15,52 @@ dev: build
 	@./bitcaskd --version
 
 build: clean generate
-	@go build \
+	@$(GOCMD) build \
 		-tags "netgo static_build" -installsuffix netgo \
-		-ldflags "-w -X $(shell go list)/internal.Version=$(VERSION) -X $(shell go list)/internal.Commit=$(COMMIT)" \
+		-ldflags "-w -X $(shell go list)/internal.Version=$(VERSION) -X $(shell go list)/internal.Commit=$(COMMIT) -X $(shell go list)/internal.Build=$(BUILD)" \
 		./cmd/bitcask/...
-	@go build \
+	@$(GOCMD) build \
 		-tags "netgo static_build" -installsuffix netgo \
-		-ldflags "-w -X $(shell go list)/internal.Version=$(VERSION) -X $(shell go list)/internal.Commit=$(COMMIT)" \
+		-ldflags "-w -X $(shell go list)/internal.Version=$(VERSION) -X $(shell go list)/internal.Commit=$(COMMIT) -X $(shell go list)/internal.Build=$(BUILD)" \
 		./cmd/bitcaskd/...
 
 generate:
-	@go generate $(shell go list)/...
+	@$(GOCMD) generate $(shell go list)/...
 
 install: build
-	@go install ./cmd/bitcask/...
-	@go install ./cmd/bitcaskd/...
+	@install -D -m 755 bitcask $(DESTDIR)/bitcask
+	@install -D -m 755 bitcaskd $(DESTDIR)/bitcaskd
 
+ifeq ($(PUBLISH), 1)
 image:
-	@docker build -t prologic/bitcask .
+	@docker build --build-arg VERSION="$(VERSION)" --build-arg COMMIT="$(COMMIT)" -t prologic/bitcask .
+	@docker push prologic/bitcask
+else
+image:
+	@docker build --build-arg VERSION="$(VERSION)" --build-arg COMMIT="$(COMMIT)" -t prologic/bitcask .
+endif
 
 release:
 	@./tools/release.sh
 
 profile: build
-	@go test -cpuprofile cpu.prof -memprofile mem.prof -v -bench .
+	@$(GOCMD) test -cpuprofile cpu.prof -memprofile mem.prof -v -bench .
 
 bench: build
-	@go test -v -run=XXX -benchmem -bench=. .
+	@$(GOCMD) test -v -run=XXX -benchmem -bench=. .
 
 mocks:
 	@mockery -all -case underscore -output ./internal/mocks -recursive
 
 test: build
-	@go test -v \
+	@$(GOCMD) test -v \
 		-cover -coverprofile=coverage.txt -covermode=atomic \
 		-coverpkg=$(shell go list) \
 		-race \
 		.
 
 setup:
-	@go get github.com/vektra/mockery/...
+	@$(GOCMD) get github.com/vektra/mockery/...
 
 clean:
 	@git clean -f -d -X

bitcask.go

@@ -2,7 +2,6 @@ package bitcask
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"hash/crc32"
 	"io"
@@ -33,41 +32,6 @@ const (
 	ttlIndexFile = "ttl_index"
 )
 
-var (
-	// ErrKeyNotFound is the error returned when a key is not found
-	ErrKeyNotFound = errors.New("error: key not found")
-
-	// ErrKeyTooLarge is the error returned for a key that exceeds the
-	// maximum allowed key size (configured with WithMaxKeySize).
-	ErrKeyTooLarge = errors.New("error: key too large")
-
-	// ErrKeyExpired is the error returned when a key is queried which has
-	// already expired (due to ttl)
-	ErrKeyExpired = errors.New("error: key expired")
-
-	// ErrEmptyKey is the error returned for a value with an empty key.
-	ErrEmptyKey = errors.New("error: empty key")
-
-	// ErrValueTooLarge is the error returned for a value that exceeds the
-	// maximum allowed value size (configured with WithMaxValueSize).
-	ErrValueTooLarge = errors.New("error: value too large")
-
-	// ErrChecksumFailed is the error returned if a key/value retrieved does
-	// not match its CRC checksum
-	ErrChecksumFailed = errors.New("error: checksum failed")
-
-	// ErrDatabaseLocked is the error returned if the database is locked
-	// (typically opened by another process)
-	ErrDatabaseLocked = errors.New("error: database locked")
-
-	ErrInvalidRange   = errors.New("error: invalid range")
-	ErrInvalidVersion = errors.New("error: invalid db version")
-
-	// ErrMergeInProgress is the error returned if merge is called when already a merge
-	// is in progress
-	ErrMergeInProgress = errors.New("error: merge already in progress")
-)
-
 // Bitcask is a struct that represents a on-disk LSM and WAL data structure
 // and in-memory hash of key/value pairs as per the Bitcask paper and seen
 // in the Riak database.
@@ -300,7 +264,9 @@ func (b *Bitcask) Sift(f func(key []byte) (bool, error)) (err error) {
 		return true
 	})
 	b.mu.RUnlock()
-
+	if err != nil {
+		return
+	}
 	b.mu.Lock()
 	defer b.mu.Unlock()
 	keysToDelete.ForEach(func(node art.Node) (cont bool) {
@@ -379,6 +345,10 @@ func (b *Bitcask) SiftScan(prefix []byte, f func(key []byte) (bool, error)) (err
 	})
 	b.mu.RUnlock()
 
+	if err != nil {
+		return
+	}
+
 	b.mu.Lock()
 	defer b.mu.Unlock()
 	keysToDelete.ForEach(func(node art.Node) (cont bool) {
@@ -458,6 +428,10 @@ func (b *Bitcask) SiftRange(start, end []byte, f func(key []byte) (bool, error))
 	})
 	b.mu.RUnlock()
 
+	if err != nil {
+		return
+	}
+
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
@@ -864,7 +838,7 @@ func Open(path string, options ...Option) (*Bitcask, error) {
 	if internal.Exists(configPath) {
 		cfg, err = config.Load(configPath)
 		if err != nil {
-			return nil, err
+			return nil, &ErrBadConfig{err}
 		}
 	} else {
 		cfg = newDefaultConfig()
@@ -886,7 +860,7 @@ func Open(path string, options ...Option) (*Bitcask, error) {
 
 	meta, err = loadMetadata(path)
 	if err != nil {
-		return nil, err
+		return nil, &ErrBadMetadata{err}
 	}
 
 	bitcask := &Bitcask{

cmd/bitcaskd/server.go

@@ -84,10 +84,41 @@ func (s *server) handleGet(cmd redcon.Command, conn redcon.Conn) {
 }
 
 func (s *server) handleKeys(cmd redcon.Command, conn redcon.Conn) {
-	conn.WriteArray(s.db.Len())
-	for key := range s.db.Keys() {
-		conn.WriteBulk(key)
+	if len(cmd.Args) != 2 {
+		conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
+		return
 	}
+
+	pattern := string(cmd.Args[1])
+
+	// Fast-track condition for improved speed
+	if pattern == "*" {
+		conn.WriteArray(s.db.Len())
+		for key := range s.db.Keys() {
+			conn.WriteBulk(key)
+		}
+		return
+	}
+
+	// Prefix handling
+	if strings.Count(pattern, "*") == 1 && strings.HasSuffix(pattern, "*") {
+		prefix := strings.ReplaceAll(pattern, "*", "")
+		count := 0
+		keys := make([][]byte, 0)
+		s.db.Scan([]byte(prefix), func(key []byte) error {
+			keys = append(keys, key)
+			count++
+			return nil
+		})
+		conn.WriteArray(count)
+		for _, key := range keys {
+			conn.WriteBulk(key)
+		}
+		return
+	}
+
+	// No results means empty array
+	conn.WriteArray(0)
 }
 
 func (s *server) handleExists(cmd redcon.Command, conn redcon.Conn) {

cmd/bitcaskd/server_test.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"net"
+	"strconv"
+	"testing"
+
+	"github.com/tidwall/redcon"
+)
+
+func TestHandleKeys(t *testing.T) {
+	s, err := newServer(":61234", "./test.db")
+	if err != nil {
+		t.Fatalf("Unable to create server: %v", err)
+	}
+	s.db.Put([]byte("foo"), []byte("bar"))
+	testCases := []TestCase{
+		{
+			Command: redcon.Command{
+				Raw:  []byte("KEYS *"),
+				Args: [][]byte{[]byte("KEYS"), []byte("*")},
+			},
+			Expected: "1,foo",
+		},
+		{
+			Command: redcon.Command{
+				Raw:  []byte("KEYS fo*"),
+				Args: [][]byte{[]byte("KEYS"), []byte("fo*")},
+			},
+			Expected: "1,foo",
+		},
+		{
+			Command: redcon.Command{
+				Raw:  []byte("KEYS ba*"),
+				Args: [][]byte{[]byte("KEYS"), []byte("ba*")},
+			},
+			Expected: "0",
+		},
+		{
+			Command: redcon.Command{
+				Raw:  []byte("KEYS *oo"),
+				Args: [][]byte{[]byte("KEYS"), []byte("*oo")},
+			},
+			Expected: "0",
+		},
+	}
+	for _, testCase := range testCases {
+		conn := DummyConn{}
+		s.handleKeys(testCase.Command, &conn)
+		if testCase.Expected != conn.Result {
+			t.Fatalf("s.handleKeys failed: expected '%s', got '%s'", testCase.Expected, conn.Result)
+		}
+	}
+}
+
+type TestCase struct {
+	Command  redcon.Command
+	Expected string
+}
+
+type DummyConn struct {
+	Result string
+}
+
+func (dc *DummyConn) RemoteAddr() string {
+	return ""
+}
+func (dc *DummyConn) Close() error {
+	return nil
+}
+func (dc *DummyConn) WriteError(msg string)  {}
+func (dc *DummyConn) WriteString(str string) {}
+func (dc *DummyConn) WriteBulk(bulk []byte) {
+	dc.Result += "," + string(bulk)
+}
+func (dc *DummyConn) WriteBulkString(bulk string) {}
+func (dc *DummyConn) WriteInt(num int)            {}
+func (dc *DummyConn) WriteInt64(num int64)        {}
+func (dc *DummyConn) WriteUint64(num uint64)      {}
+func (dc *DummyConn) WriteArray(count int) {
+	dc.Result = strconv.Itoa(count)
+}
+func (dc *DummyConn) WriteNull()               {}
+func (dc *DummyConn) WriteRaw(data []byte)     {}
+func (dc *DummyConn) WriteAny(any interface{}) {}
+func (dc *DummyConn) Context() interface{} {
+	return nil
+}
+func (dc *DummyConn) SetContext(v interface{}) {}
+func (dc *DummyConn) SetReadBuffer(bytes int)  {}
+func (dc *DummyConn) Detach() redcon.DetachedConn {
+	return nil
+}
+func (dc *DummyConn) ReadPipeline() []redcon.Command {
+	return nil
+}
+func (dc *DummyConn) PeekPipeline() []redcon.Command {
+	return nil
+}
+func (dc *DummyConn) NetConn() net.Conn {
+	return nil
+}

errors.go

@@ -0,0 +1,77 @@
+package bitcask
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	// ErrKeyNotFound is the error returned when a key is not found
+	ErrKeyNotFound = errors.New("error: key not found")
+
+	// ErrKeyTooLarge is the error returned for a key that exceeds the
+	// maximum allowed key size (configured with WithMaxKeySize).
+	ErrKeyTooLarge = errors.New("error: key too large")
+
+	// ErrKeyExpired is the error returned when a key is queried which has
+	// already expired (due to ttl)
+	ErrKeyExpired = errors.New("error: key expired")
+
+	// ErrEmptyKey is the error returned for a value with an empty key.
+	ErrEmptyKey = errors.New("error: empty key")
+
+	// ErrValueTooLarge is the error returned for a value that exceeds the
+	// maximum allowed value size (configured with WithMaxValueSize).
+	ErrValueTooLarge = errors.New("error: value too large")
+
+	// ErrChecksumFailed is the error returned if a key/value retrieved does
+	// not match its CRC checksum
+	ErrChecksumFailed = errors.New("error: checksum failed")
+
+	// ErrDatabaseLocked is the error returned if the database is locked
+	// (typically opened by another process)
+	ErrDatabaseLocked = errors.New("error: database locked")
+
+	// ErrInvalidRange is the error returned when the range scan is invalid
+	ErrInvalidRange = errors.New("error: invalid range")
+
+	// ErrInvalidVersion is the error returned when the database version is invalid
+	ErrInvalidVersion = errors.New("error: invalid db version")
+
+	// ErrMergeInProgress is the error returned if merge is called when already a merge
+	// is in progress
+	ErrMergeInProgress = errors.New("error: merge already in progress")
+)
+
+// ErrBadConfig is the error returned on failure to load the database config
+type ErrBadConfig struct {
+	Err error
+}
+
+func (e *ErrBadConfig) Is(target error) bool {
+	if _, ok := target.(*ErrBadConfig); ok {
+		return true
+	}
+	return errors.Is(e.Err, target)
+}
+func (e *ErrBadConfig) Unwrap() error { return e.Err }
+func (e *ErrBadConfig) Error() string {
+	return fmt.Sprintf("error reading config.json: %s", e.Err)
+}
+
+// ErrBadMetadata is the error returned on failure to load the database metadata
+type ErrBadMetadata struct {
+	Err error
+}
+
+func (e *ErrBadMetadata) Is(target error) bool {
+	if _, ok := target.(*ErrBadMetadata); ok {
+		return true
+	}
+	return errors.Is(e.Err, target)
+}
+
+func (e *ErrBadMetadata) Unwrap() error { return e.Err }
+func (e *ErrBadMetadata) Error() string {
+	return fmt.Sprintf("error reading meta.json: %s", e.Err)
+}

internal/data/datafile.go

@@ -6,9 +6,9 @@ import (
 	"path/filepath"
 	"sync"
 
-	"github.com/pkg/errors"
 	"git.mills.io/prologic/bitcask/internal"
 	"git.mills.io/prologic/bitcask/internal/data/codec"
+	"github.com/pkg/errors"
 	"golang.org/x/exp/mmap"
 )
 
@@ -74,9 +74,11 @@ func NewDatafile(path string, id int, readonly bool, maxKeySize uint32, maxValue
 		return nil, errors.Wrap(err, "error calling Stat()")
 	}
 
-	ra, err = mmap.Open(fn)
-	if err != nil {
-		return nil, err
+	if readonly {
+		ra, err = mmap.Open(fn)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	offset := stat.Size()
@@ -107,7 +109,9 @@ func (df *datafile) Name() string {
 
 func (df *datafile) Close() error {
 	defer func() {
-		df.ra.Close()
+		if df.ra != nil {
+			df.ra.Close()
+		}
 		df.r.Close()
 	}()
 
@@ -155,7 +159,10 @@ func (df *datafile) ReadAt(index, size int64) (e internal.Entry, err error) {
 
 	b := make([]byte, size)
 
-	if df.w == nil {
+	df.RLock()
+	defer df.RUnlock()
+
+	if df.ra != nil {
 		n, err = df.ra.ReadAt(b, index)
 	} else {
 		n, err = df.r.ReadAt(b, index)

internal/version.go

@@ -2,17 +2,49 @@ package internal
 
 import (
 	"fmt"
+	"runtime/debug"
+	"strings"
+)
+
+const (
+	defaultVersion = "0.0.0"
+	defaultCommit  = "HEAD"
+	defaultBuild   = "0000-01-01:00:00+00:00"
 )
 
 var (
-	// Version release version
-	Version = "0.0.1"
+	// Version is the tagged release version in the form <major>.<minor>.<patch>
+	// following semantic versioning and is overwritten by the build system.
+	Version = defaultVersion
 
-	// Commit will be overwritten automatically by the build system
-	Commit = "HEAD"
+	// Commit is the commit sha of the build (normally from Git) and is overwritten
+	// by the build system.
+	Commit = defaultCommit
+
+	// Build is the date and time of the build as an RFC3339 formatted string
+	// and is overwritten by the build system.
+	Build = defaultBuild
 )
 
-// FullVersion returns the full version and commit hash
+// FullVersion display the full version and build
 func FullVersion() string {
-	return fmt.Sprintf("%s@%s", Version, Commit)
+	var sb strings.Builder
+
+	isDefault := Version == defaultVersion && Commit == defaultCommit && Build == defaultBuild
+
+	if !isDefault {
+		sb.WriteString(fmt.Sprintf("%s@%s %s", Version, Commit, Build))
+	}
+
+	if info, ok := debug.ReadBuildInfo(); ok {
+		if isDefault {
+			sb.WriteString(fmt.Sprintf(" %s", info.Main.Version))
+		}
+		sb.WriteString(fmt.Sprintf(" %s", info.GoVersion))
+		if info.Main.Sum != "" {
+			sb.WriteString(fmt.Sprintf(" %s", info.Main.Sum))
+		}
+	}
+
+	return sb.String()
 }

tools/release.sh

@@ -26,7 +26,7 @@ echo "Releasing ${TAG} ..."
 git-chglog --next-tag="${TAG}" --output CHANGELOG.md
 git commit -a -m "Update CHANGELOG for ${TAG}"
 git tag -a -s -m "Release ${TAG}" "${TAG}"
-git push --tags
+git push && git push --tags
 goreleaser release \
   --rm-dist \
   --release-notes <(git-chglog "${TAG}" | tail -n+5)