Update CHANGELOG for v1.0.2

cc @taigrr

This PR will _hopefully_ help to fix some critical isseus in the real world with several or more [Yarn.social](https://yarn.social) pods running [yarnd](https://git.mills.io/yarnsocial/yarn) where starting back up after a power failure or crash can sometimes result in an empty `config.json` or empty `meta.json` or both!

I'm not actually sure how this can arise, and as yet I haven't been able to reproduce it (_I can only assume this has to be failures cases outside of our control_); but in any case the application and database is recoverable by simply `rm config.json` and/or `rm meta.json`.

So this PR makes errors loading the config and metadata first-class errors and exported error types that consumers of the library can use to perform automated recovery without requiring human intervention.

Basiclaly in this case it's no big deal we lost the database config of metadata, we can simply carry on.

Co-authored-by: James Mills <prologic@shortcircuit.net.au>
Reviewed-on: https://git.mills.io/prologic/bitcask/pulls/241
Co-authored-by: James Mills <james@mills.io>
Co-committed-by: James Mills <james@mills.io>

.gitignore

@@ -5,6 +5,7 @@
 
 /tmp
 /dist
+/cacheDb
 /coverage.txt
 
 /bitcask

AUTHORS

@@ -17,3 +17,5 @@ Yash Chandra <yashschandra@gmail.com>
 Yury Fedorov orlangure
 o2gy84 <o2gy84@gmail.com>
 garsue <labs.garsue@gmail.com>
+biozz <ielfimov@gmail.com>
+jason3gb <jason3gb@gmail.com>

CHANGELOG.md

@@ -1,4 +1,44 @@
 
+<a name="v1.0.2"></a>
+## [v1.0.2](https://git.mills.io/prologic/bitcask/compare/v1.0.1...v1.0.2) (2021-11-01)
+
+### Bug Fixes
+
+* Fix a data race in Datafile.ReadAt()
+* Fix release tool
+
+
+<a name="v1.0.1"></a>
+## [v1.0.1](https://git.mills.io/prologic/bitcask/compare/v1.0.0...v1.0.1) (2021-10-31)
+
+### Features
+
+* Add ErrBadConfig and ErrBadMetadata as errors that consumers can check and use (#241)
+* Add key prefix matching to KEYS command (#237)
+
+### Updates
+
+* Update CHANGELOG for v1.0.1
+* Update image target
+
+
+<a name="v1.0.0"></a>
+## [v1.0.0](https://git.mills.io/prologic/bitcask/compare/1.0.0...v1.0.0) (2021-07-24)
+
+### Updates
+
+* Update CHANGELOG for v1.0.0
+
+
+<a name="1.0.0"></a>
+## [1.0.0](https://git.mills.io/prologic/bitcask/compare/v0.3.14...1.0.0) (2021-07-24)
+
+### Updates
+
+* Update CHANGELOG for 1.0.0
+* Update README
+
+
 <a name="v0.3.14"></a>
 ## [v0.3.14](https://git.mills.io/prologic/bitcask/compare/v0.3.13...v0.3.14) (2021-07-21)
 
@@ -14,6 +54,10 @@
 
 * Add RangeScan() support (#160)
 
+### Updates
+
+* Update CHANGELOG for v0.3.14
+
 
 <a name="v0.3.13"></a>
 ## [v0.3.13](https://git.mills.io/prologic/bitcask/compare/v0.3.12...v0.3.13) (2021-07-16)

Makefile

@@ -27,8 +27,14 @@ install: build
 	@go install ./cmd/bitcask/...
 	@go install ./cmd/bitcaskd/...
 
+ifeq ($(PUBLISH), 1)
 image:
-	@docker build -t prologic/bitcask .
+	@docker build --build-arg VERSION="$(VERSION)" --build-arg COMMIT="$(COMMIT)" -t prologic/bitcask .
+	@docker push prologic/bitcask
+else
+image:
+	@docker build --build-arg VERSION="$(VERSION)" --build-arg COMMIT="$(COMMIT)" -t prologic/bitcask .
+endif
 
 release:
 	@./tools/release.sh

README.md

@@ -220,24 +220,14 @@ Support the ongoing development of Bitcask!
 
 - Become a [Sponsor](https://www.patreon.com/prologic)
 
-## Stargazers over time
-
-[![Stargazers over time](https://starcharts.herokuapp.com/prologic/bitcask.svg)](https://starcharts.herokuapp.com/prologic/bitcask)
-
 ## Contributors
 
-Thank you to all those that have contributed to this project, battle-tested it, used it in their own projects or products, fixed bugs, improved performance and even fix tiny typos in documentation! Thank you and keep contributing!
+Thank you to all those that have contributed to this project, battle-tested it,
+used it in their own projects or products, fixed bugs, improved performance
+and even fix tiny typos in documentation! Thank you and keep contributing!
 
-You can find an [AUTHORS](/AUTHORS) file where we keep a list of contributors to the project. If you contribute a PR please consider adding your name there. There is also GitHub's own [Contributors](https://git.mills.io/prologic/bitcask/graphs/contributors) statistics.
-
-[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/0)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/0)
-[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/1)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/1)
-[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/2)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/2)
-[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/3)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/3)
-[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/4)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/4)
-[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/5)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/5)
-[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/6)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/6)
-[![](https://sourcerer.io/fame/prologic/prologic/bitcask/images/7)](https://sourcerer.io/fame/prologic/prologic/bitcask/links/7)
+You can find an [AUTHORS](/AUTHORS) file where we keep a list of contributors
+to the project. If you contribute a PR please consider adding your name there.
 
 ## Related Projects
 

bitcask.go

@@ -2,7 +2,6 @@ package bitcask
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"hash/crc32"
 	"io"
@@ -33,41 +32,6 @@ const (
 	ttlIndexFile = "ttl_index"
 )
 
-var (
-	// ErrKeyNotFound is the error returned when a key is not found
-	ErrKeyNotFound = errors.New("error: key not found")
-
-	// ErrKeyTooLarge is the error returned for a key that exceeds the
-	// maximum allowed key size (configured with WithMaxKeySize).
-	ErrKeyTooLarge = errors.New("error: key too large")
-
-	// ErrKeyExpired is the error returned when a key is queried which has
-	// already expired (due to ttl)
-	ErrKeyExpired = errors.New("error: key expired")
-
-	// ErrEmptyKey is the error returned for a value with an empty key.
-	ErrEmptyKey = errors.New("error: empty key")
-
-	// ErrValueTooLarge is the error returned for a value that exceeds the
-	// maximum allowed value size (configured with WithMaxValueSize).
-	ErrValueTooLarge = errors.New("error: value too large")
-
-	// ErrChecksumFailed is the error returned if a key/value retrieved does
-	// not match its CRC checksum
-	ErrChecksumFailed = errors.New("error: checksum failed")
-
-	// ErrDatabaseLocked is the error returned if the database is locked
-	// (typically opened by another process)
-	ErrDatabaseLocked = errors.New("error: database locked")
-
-	ErrInvalidRange   = errors.New("error: invalid range")
-	ErrInvalidVersion = errors.New("error: invalid db version")
-
-	// ErrMergeInProgress is the error returned if merge is called when already a merge
-	// is in progress
-	ErrMergeInProgress = errors.New("error: merge already in progress")
-)
-
 // Bitcask is a struct that represents a on-disk LSM and WAL data structure
 // and in-memory hash of key/value pairs as per the Bitcask paper and seen
 // in the Riak database.
@@ -864,7 +828,7 @@ func Open(path string, options ...Option) (*Bitcask, error) {
 	if internal.Exists(configPath) {
 		cfg, err = config.Load(configPath)
 		if err != nil {
-			return nil, err
+			return nil, &ErrBadConfig{err}
 		}
 	} else {
 		cfg = newDefaultConfig()
@@ -886,7 +850,7 @@ func Open(path string, options ...Option) (*Bitcask, error) {
 
 	meta, err = loadMetadata(path)
 	if err != nil {
-		return nil, err
+		return nil, &ErrBadMetadata{err}
 	}
 
 	bitcask := &Bitcask{

cmd/bitcaskd/server.go

@@ -84,10 +84,41 @@ func (s *server) handleGet(cmd redcon.Command, conn redcon.Conn) {
 }
 
 func (s *server) handleKeys(cmd redcon.Command, conn redcon.Conn) {
-	conn.WriteArray(s.db.Len())
-	for key := range s.db.Keys() {
-		conn.WriteBulk(key)
+	if len(cmd.Args) != 2 {
+		conn.WriteError("ERR wrong number of arguments for '" + string(cmd.Args[0]) + "' command")
+		return
 	}
+
+	pattern := string(cmd.Args[1])
+
+	// Fast-track condition for improved speed
+	if pattern == "*" {
+		conn.WriteArray(s.db.Len())
+		for key := range s.db.Keys() {
+			conn.WriteBulk(key)
+		}
+		return
+	}
+
+	// Prefix handling
+	if strings.Count(pattern, "*") == 1 && strings.HasSuffix(pattern, "*") {
+		prefix := strings.ReplaceAll(pattern, "*", "")
+		count := 0
+		keys := make([][]byte, 0)
+		s.db.Scan([]byte(prefix), func(key []byte) error {
+			keys = append(keys, key)
+			count++
+			return nil
+		})
+		conn.WriteArray(count)
+		for _, key := range keys {
+			conn.WriteBulk(key)
+		}
+		return
+	}
+
+	// No results means empty array
+	conn.WriteArray(0)
 }
 
 func (s *server) handleExists(cmd redcon.Command, conn redcon.Conn) {

cmd/bitcaskd/server_test.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"net"
+	"strconv"
+	"testing"
+
+	"github.com/tidwall/redcon"
+)
+
+func TestHandleKeys(t *testing.T) {
+	s, err := newServer(":61234", "./test.db")
+	if err != nil {
+		t.Fatalf("Unable to create server: %v", err)
+	}
+	s.db.Put([]byte("foo"), []byte("bar"))
+	testCases := []TestCase{
+		{
+			Command: redcon.Command{
+				Raw:  []byte("KEYS *"),
+				Args: [][]byte{[]byte("KEYS"), []byte("*")},
+			},
+			Expected: "1,foo",
+		},
+		{
+			Command: redcon.Command{
+				Raw:  []byte("KEYS fo*"),
+				Args: [][]byte{[]byte("KEYS"), []byte("fo*")},
+			},
+			Expected: "1,foo",
+		},
+		{
+			Command: redcon.Command{
+				Raw:  []byte("KEYS ba*"),
+				Args: [][]byte{[]byte("KEYS"), []byte("ba*")},
+			},
+			Expected: "0",
+		},
+		{
+			Command: redcon.Command{
+				Raw:  []byte("KEYS *oo"),
+				Args: [][]byte{[]byte("KEYS"), []byte("*oo")},
+			},
+			Expected: "0",
+		},
+	}
+	for _, testCase := range testCases {
+		conn := DummyConn{}
+		s.handleKeys(testCase.Command, &conn)
+		if testCase.Expected != conn.Result {
+			t.Fatalf("s.handleKeys failed: expected '%s', got '%s'", testCase.Expected, conn.Result)
+		}
+	}
+}
+
+type TestCase struct {
+	Command  redcon.Command
+	Expected string
+}
+
+type DummyConn struct {
+	Result string
+}
+
+func (dc *DummyConn) RemoteAddr() string {
+	return ""
+}
+func (dc *DummyConn) Close() error {
+	return nil
+}
+func (dc *DummyConn) WriteError(msg string)  {}
+func (dc *DummyConn) WriteString(str string) {}
+func (dc *DummyConn) WriteBulk(bulk []byte) {
+	dc.Result += "," + string(bulk)
+}
+func (dc *DummyConn) WriteBulkString(bulk string) {}
+func (dc *DummyConn) WriteInt(num int)            {}
+func (dc *DummyConn) WriteInt64(num int64)        {}
+func (dc *DummyConn) WriteUint64(num uint64)      {}
+func (dc *DummyConn) WriteArray(count int) {
+	dc.Result = strconv.Itoa(count)
+}
+func (dc *DummyConn) WriteNull()               {}
+func (dc *DummyConn) WriteRaw(data []byte)     {}
+func (dc *DummyConn) WriteAny(any interface{}) {}
+func (dc *DummyConn) Context() interface{} {
+	return nil
+}
+func (dc *DummyConn) SetContext(v interface{}) {}
+func (dc *DummyConn) SetReadBuffer(bytes int)  {}
+func (dc *DummyConn) Detach() redcon.DetachedConn {
+	return nil
+}
+func (dc *DummyConn) ReadPipeline() []redcon.Command {
+	return nil
+}
+func (dc *DummyConn) PeekPipeline() []redcon.Command {
+	return nil
+}
+func (dc *DummyConn) NetConn() net.Conn {
+	return nil
+}

errors.go

@@ -0,0 +1,77 @@
+package bitcask
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	// ErrKeyNotFound is the error returned when a key is not found
+	ErrKeyNotFound = errors.New("error: key not found")
+
+	// ErrKeyTooLarge is the error returned for a key that exceeds the
+	// maximum allowed key size (configured with WithMaxKeySize).
+	ErrKeyTooLarge = errors.New("error: key too large")
+
+	// ErrKeyExpired is the error returned when a key is queried which has
+	// already expired (due to ttl)
+	ErrKeyExpired = errors.New("error: key expired")
+
+	// ErrEmptyKey is the error returned for a value with an empty key.
+	ErrEmptyKey = errors.New("error: empty key")
+
+	// ErrValueTooLarge is the error returned for a value that exceeds the
+	// maximum allowed value size (configured with WithMaxValueSize).
+	ErrValueTooLarge = errors.New("error: value too large")
+
+	// ErrChecksumFailed is the error returned if a key/value retrieved does
+	// not match its CRC checksum
+	ErrChecksumFailed = errors.New("error: checksum failed")
+
+	// ErrDatabaseLocked is the error returned if the database is locked
+	// (typically opened by another process)
+	ErrDatabaseLocked = errors.New("error: database locked")
+
+	// ErrInvalidRange is the error returned when the range scan is invalid
+	ErrInvalidRange = errors.New("error: invalid range")
+
+	// ErrInvalidVersion is the error returned when the database version is invalid
+	ErrInvalidVersion = errors.New("error: invalid db version")
+
+	// ErrMergeInProgress is the error returned if merge is called when already a merge
+	// is in progress
+	ErrMergeInProgress = errors.New("error: merge already in progress")
+)
+
+// ErrBadConfig is the error returned on failure to load the database config
+type ErrBadConfig struct {
+	Err error
+}
+
+func (e *ErrBadConfig) Is(target error) bool {
+	if _, ok := target.(*ErrBadConfig); ok {
+		return true
+	}
+	return errors.Is(e.Err, target)
+}
+func (e *ErrBadConfig) Unwrap() error { return e.Err }
+func (e *ErrBadConfig) Error() string {
+	return fmt.Sprintf("error reading config.json: %s", e.Err)
+}
+
+// ErrBadMetadata is the error returned on failure to load the database metadata
+type ErrBadMetadata struct {
+	Err error
+}
+
+func (e *ErrBadMetadata) Is(target error) bool {
+	if _, ok := target.(*ErrBadMetadata); ok {
+		return true
+	}
+	return errors.Is(e.Err, target)
+}
+
+func (e *ErrBadMetadata) Unwrap() error { return e.Err }
+func (e *ErrBadMetadata) Error() string {
+	return fmt.Sprintf("error reading meta.json: %s", e.Err)
+}

internal/data/datafile.go

@@ -6,9 +6,9 @@ import (
 	"path/filepath"
 	"sync"
 
-	"github.com/pkg/errors"
 	"git.mills.io/prologic/bitcask/internal"
 	"git.mills.io/prologic/bitcask/internal/data/codec"
+	"github.com/pkg/errors"
 	"golang.org/x/exp/mmap"
 )
 
@@ -74,9 +74,11 @@ func NewDatafile(path string, id int, readonly bool, maxKeySize uint32, maxValue
 		return nil, errors.Wrap(err, "error calling Stat()")
 	}
 
-	ra, err = mmap.Open(fn)
-	if err != nil {
-		return nil, err
+	if readonly {
+		ra, err = mmap.Open(fn)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	offset := stat.Size()
@@ -107,7 +109,9 @@ func (df *datafile) Name() string {
 
 func (df *datafile) Close() error {
 	defer func() {
-		df.ra.Close()
+		if df.ra != nil {
+			df.ra.Close()
+		}
 		df.r.Close()
 	}()
 
@@ -155,7 +159,10 @@ func (df *datafile) ReadAt(index, size int64) (e internal.Entry, err error) {
 
 	b := make([]byte, size)
 
-	if df.w == nil {
+	df.RLock()
+	defer df.RUnlock()
+
+	if df.ra != nil {
 		n, err = df.ra.ReadAt(b, index)
 	} else {
 		n, err = df.r.ReadAt(b, index)

tools/release.sh

@@ -26,7 +26,7 @@ echo "Releasing ${TAG} ..."
 git-chglog --next-tag="${TAG}" --output CHANGELOG.md
 git commit -a -m "Update CHANGELOG for ${TAG}"
 git tag -a -s -m "Release ${TAG}" "${TAG}"
-git push --tags
+git push && git push --tags
 goreleaser release \
   --rm-dist \
   --release-notes <(git-chglog "${TAG}" | tail -n+5)