local live backup support (#185)

* live backup first commit

* exclude lock file in backup

* create path if not exist for backup

Co-authored-by: yash <yash.chandra@grabpay.com>
Co-authored-by: James Mills <prologic@shortcircuit.net.au>

internal/metadata/metadata.go

@@ -0,0 +1,21 @@
+package metadata
+
+import (
+	"os"
+
+	"github.com/prologic/bitcask/internal"
+)
+
+type MetaData struct {
+	IndexUpToDate bool `json:"index_up_to_date"`
+}
+
+func (m *MetaData) Save(path string, mode os.FileMode) error {
+	return internal.SaveJsonToFile(m, path, mode)
+}
+
+func Load(path string) (*MetaData, error) {
+	var m MetaData
+	err := internal.LoadFromJsonFile(path, &m)
+	return &m, err
+}

internal/utils.go

@@ -1,7 +1,9 @@
 package internal
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
@@ -63,3 +65,48 @@ func ParseIds(fns []string) ([]int, error) {
 	sort.Ints(ids)
 	return ids, nil
 }
+
+// Copy copies source contents to destination
+func Copy(src, dst string, exclude []string) error {
+	return filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+		relPath := strings.Replace(path, src, "", 1)
+		if relPath == "" {
+			return nil
+		}
+		for _, e := range exclude {
+			matched, err := filepath.Match(e, info.Name())
+			if err != nil {
+				return err
+			}
+			if matched {
+				return nil
+			}
+		}
+		if info.IsDir() {
+			return os.Mkdir(filepath.Join(dst, relPath), info.Mode())
+		}
+		var data, err1 = ioutil.ReadFile(filepath.Join(src, relPath))
+		if err1 != nil {
+			return err1
+		}
+		return ioutil.WriteFile(filepath.Join(dst, relPath), data, info.Mode())
+	})
+}
+
+// SaveJsonToFile converts v into json and store in file identified by path
+func SaveJsonToFile(v interface{}, path string, mode os.FileMode) error {
+	b, err := json.Marshal(v)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(path, b, mode)
+}
+
+// LoadFromJsonFile reads file located at `path` and put its content in json format in v
+func LoadFromJsonFile(path string, v interface{}) error {
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(b, v)
+}

internal/utils_test.go

@@ -0,0 +1,108 @@
+package internal
+
+import (
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_Copy(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("CopyDir", func(t *testing.T) {
+		tempsrc, err := ioutil.TempDir("", "test")
+		assert.NoError(err)
+		defer os.RemoveAll(tempsrc)
+		var f *os.File
+
+		tempdir, err := ioutil.TempDir(tempsrc, "")
+		assert.NoError(err)
+
+		f, err = os.OpenFile(filepath.Join(tempsrc, "file1"), os.O_WRONLY|os.O_CREATE, 0755)
+		assert.NoError(err)
+		n, err := f.WriteString("test123")
+		assert.Equal(7, n)
+		assert.NoError(err)
+		f.Close()
+
+		f, err = os.OpenFile(filepath.Join(tempsrc, "file2"), os.O_WRONLY|os.O_CREATE, 0755)
+		assert.NoError(err)
+		n, err = f.WriteString("test1234")
+		assert.Equal(8, n)
+		assert.NoError(err)
+		f.Close()
+
+		f, err = os.OpenFile(filepath.Join(tempsrc, "file3"), os.O_WRONLY|os.O_CREATE, 0755)
+		assert.NoError(err)
+		f.Close()
+
+		tempdst, err := ioutil.TempDir("", "backup")
+		assert.NoError(err)
+		defer os.RemoveAll(tempdst)
+		err = Copy(tempsrc, tempdst, []string{"file3"})
+		assert.NoError(err)
+		buf := make([]byte, 10)
+
+		exists := Exists(filepath.Join(tempdst, filepath.Base(tempdir)))
+		assert.Equal(true, exists)
+
+		f, err = os.Open(filepath.Join(tempdst, "file1"))
+		assert.NoError(err)
+		n, err = f.Read(buf[:7])
+		assert.NoError(err)
+		assert.Equal(7, n)
+		assert.Equal([]byte("test123"), buf[:7])
+		_, err = f.Read(buf)
+		assert.Equal(io.EOF, err)
+		f.Close()
+
+		f, err = os.Open(filepath.Join(tempdst, "file2"))
+		assert.NoError(err)
+		n, err = f.Read(buf[:8])
+		assert.NoError(err)
+		assert.Equal(8, n)
+		assert.Equal([]byte("test1234"), buf[:8])
+		_, err = f.Read(buf)
+		assert.Equal(io.EOF, err)
+		f.Close()
+
+		exists = Exists(filepath.Join(tempdst, "file3"))
+		assert.Equal(false, exists)
+	})
+}
+
+func Test_SaveAndLoad(t *testing.T) {
+	assert := assert.New(t)
+	t.Run("save and load", func(t *testing.T) {
+		tempdir, err := ioutil.TempDir("", "bitcask")
+		assert.NoError(err)
+		defer os.RemoveAll(tempdir)
+		type test struct {
+			Value bool `json:"value"`
+		}
+		m := test{Value: true}
+		err = SaveJsonToFile(&m, filepath.Join(tempdir, "meta.json"), 0755)
+		assert.NoError(err)
+		m1 := test{}
+		err = LoadFromJsonFile(filepath.Join(tempdir, "meta.json"), &m1)
+		assert.NoError(err)
+		assert.Equal(m, m1)
+	})
+
+	t.Run("save and load error", func(t *testing.T) {
+		tempdir, err := ioutil.TempDir("", "bitcask")
+		assert.NoError(err)
+		defer os.RemoveAll(tempdir)
+		type test struct {
+			Value bool `json:"value"`
+		}
+		err = SaveJsonToFile(make(chan int), filepath.Join(tempdir, "meta.json"), 0755)
+		assert.Error(err)
+		m1 := test{}
+		err = LoadFromJsonFile(filepath.Join(tempdir, "meta.json"), &m1)
+		assert.Error(err)
+	})
+}