diff --git a/0201/cell.go b/0201/cell.go new file mode 100644 index 0000000..e756505 --- /dev/null +++ b/0201/cell.go @@ -0,0 +1,20 @@ +package db0201 + +type CellType uint8 + +const ( + TypeI64 CellType = 1 + TypeStr CellType = 2 +) + +type Cell struct { + Type CellType + I64 int64 + Str []byte +} + +func (cell *Cell) Encode(toAppend []byte) []byte + +func (cell *Cell) Decode(data []byte) (rest []byte, err error) + +// QzBQWVJJOUhU https://trialofcode.org/ diff --git a/0201/cell_test.go b/0201/cell_test.go new file mode 100644 index 0000000..7a7f0bc --- /dev/null +++ b/0201/cell_test.go @@ -0,0 +1,27 @@ +package db0201 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTableCell(t *testing.T) { + cell := Cell{Type: TypeI64, I64: -2} + data := []byte{0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + assert.Equal(t, data, cell.Encode(nil)) + decoded := Cell{Type: TypeI64} + rest, err := decoded.Decode(data) + assert.True(t, len(rest) == 0 && err == nil) + assert.Equal(t, cell, decoded) + + cell = Cell{Type: TypeStr, Str: []byte("asdf")} + data = []byte{4, 0, 0, 0, 'a', 's', 'd', 'f'} + assert.Equal(t, data, cell.Encode(nil)) + decoded = Cell{Type: TypeStr} + rest, err = decoded.Decode(data) + assert.True(t, len(rest) == 0 && err == nil) + assert.Equal(t, cell, decoded) +} + +// QzBQWVJJOUhU https://trialofcode.org/ diff --git a/0201/kv.go b/0201/kv.go new file mode 100644 index 0000000..fa8d725 --- /dev/null +++ b/0201/kv.go @@ -0,0 +1,66 @@ +package db0201 + +import ( + "bytes" +) + +type KV struct { + log Log + mem map[string][]byte +} + +func (kv *KV) Open() error { + if err := kv.log.Open(); err != nil { + return err + } + + kv.mem = map[string][]byte{} + for { + ent := Entry{} + eof, err := kv.log.Read(&ent) + if err != nil { + return err + } else if eof { + break + } + + if ent.deleted { + delete(kv.mem, string(ent.key)) + } else { + kv.mem[string(ent.key)] = ent.val + } + } + return nil +} + +func (kv *KV) Close() error { return kv.log.Close() } + +func (kv *KV) Get(key []byte) (val []byte, ok bool, err error) { + val, ok = kv.mem[string(key)] + return +} + +func (kv *KV) Set(key []byte, val []byte) (updated bool, err error) { + prev, exist := kv.mem[string(key)] + updated = !exist || !bytes.Equal(prev, val) + if updated { + if err = kv.log.Write(&Entry{key: key, val: val}); err != nil { + return false, err + } + kv.mem[string(key)] = val + } + return +} + +func (kv *KV) Del(key []byte) (deleted bool, err error) { + _, deleted = kv.mem[string(key)] + if deleted { + if err = kv.log.Write(&Entry{key: key, deleted: true}); err != nil { + return false, err + } + delete(kv.mem, string(key)) + } + return +} + +// QzBQWVJJOUhU https://trialofcode.org/ diff --git a/0201/kv_entry.go b/0201/kv_entry.go new file mode 100644 index 0000000..ea6a93b --- /dev/null +++ b/0201/kv_entry.go @@ -0,0 +1,63 @@ +package db0201 + +import ( + "encoding/binary" + "errors" + "hash/crc32" + "io" +) + +type Entry struct { + key []byte + val []byte + deleted bool +} + +func (ent *Entry) Encode() []byte { + data := make([]byte, 4+4+4+1+len(ent.key)+len(ent.val)) + binary.LittleEndian.PutUint32(data[4:8], uint32(len(ent.key))) + copy(data[4+4+4+1:], ent.key) + if ent.deleted { + data[4+4+4] = 1 + } else { + binary.LittleEndian.PutUint32(data[8:12], uint32(len(ent.val))) + copy(data[4+4+4+1+len(ent.key):], ent.val) + } + binary.LittleEndian.PutUint32(data[0:4], crc32.ChecksumIEEE(data[4:])) + return data +} + +var ErrBadSum = errors.New("bad checksum") + +func (ent *Entry) Decode(r io.Reader) error { + var header [4 + 4 + 4 + 1]byte + if _, err := io.ReadFull(r, header[:]); err != nil { + return err + } + klen := int(binary.LittleEndian.Uint32(header[4:8])) + vlen := int(binary.LittleEndian.Uint32(header[8:12])) + deleted := header[4+4+4] + + data := make([]byte, klen+vlen) + if _, err := io.ReadFull(r, data); err != nil { + return err + } + + h := crc32.NewIEEE() + h.Write(header[4:]) + h.Write(data) + if h.Sum32() != binary.LittleEndian.Uint32(header[0:4]) { + return ErrBadSum + } + + ent.key = data[:klen] + if deleted != 0 { + ent.deleted = true + } else { + ent.deleted = false + ent.val = data[klen:] + } + return nil +} + +// QzBQWVJJOUhU https://trialofcode.org/ diff --git a/0201/kv_test.go b/0201/kv_test.go new file mode 100644 index 0000000..1c08b00 --- /dev/null +++ b/0201/kv_test.go @@ -0,0 +1,126 @@ +package db0201 + +import ( + "bytes" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestKVBasic(t *testing.T) { + kv := KV{} + kv.log.FileName = ".test_db" + defer os.Remove(kv.log.FileName) + + os.Remove(kv.log.FileName) + err := kv.Open() + assert.Nil(t, err) + defer kv.Close() + + updated, err := kv.Set([]byte("k1"), []byte("v1")) + assert.True(t, updated && err == nil) + + val, ok, err := kv.Get([]byte("k1")) + assert.True(t, string(val) == "v1" && ok && err == nil) + + _, ok, err = kv.Get([]byte("xxx")) + assert.True(t, !ok && err == nil) + + updated, err = kv.Del([]byte("xxx")) + assert.True(t, !updated && err == nil) + + updated, err = kv.Del([]byte("k1")) + assert.True(t, updated && err == nil) + + _, ok, err = kv.Get([]byte("xxx")) + assert.True(t, !ok && err == nil) + + updated, err = kv.Set([]byte("k2"), []byte("v2")) + assert.True(t, updated && err == nil) + + // reopen + kv.Close() + err = kv.Open() + assert.Nil(t, err) + + _, ok, err = kv.Get([]byte("k1")) + assert.True(t, !ok && err == nil) + val, ok, err = kv.Get([]byte("k2")) + assert.True(t, string(val) == "v2" && ok && err == nil) +} + +func TestKVRecovery(t *testing.T) { + kv := KV{} + kv.log.FileName = ".test_db" + defer os.Remove(kv.log.FileName) + + prepare := func() { + os.Remove(kv.log.FileName) + + err := kv.Open() + assert.Nil(t, err) + defer kv.Close() + + updated, err := kv.Set([]byte("k1"), []byte("v1")) + assert.True(t, updated && err == nil) + updated, err = kv.Set([]byte("k2"), []byte("v2")) + assert.True(t, updated && err == nil) + } + + prepare() + // simulate truncated log + fp, _ := os.OpenFile(kv.log.FileName, os.O_RDWR, 0o644) + st, _ := fp.Stat() + fp.Truncate(st.Size() - 1) + fp.Close() + // reopen + err := kv.Open() + assert.Nil(t, err) + // test + val, ok, err := kv.Get([]byte("k1")) + assert.True(t, string(val) == "v1" && ok && err == nil) + _, ok, err = kv.Get([]byte("k2")) // bad + assert.True(t, !ok && err == nil) + kv.Close() + + prepare() + // simulate bad checksum + fp, _ = os.OpenFile(kv.log.FileName, os.O_RDWR, 0o644) + st, _ = fp.Stat() + fp.WriteAt([]byte{0}, st.Size()-1) + fp.Close() + // reopen + err = kv.Open() + assert.Nil(t, err) + // test + val, ok, err = kv.Get([]byte("k1")) + assert.True(t, string(val) == "v1" && ok && err == nil) + _, ok, err = kv.Get([]byte("k2")) // bad + assert.True(t, !ok && err == nil) + kv.Close() +} + +func TestEntryEncode(t *testing.T) { + ent := Entry{key: []byte("k1"), val: []byte("xxx")} + data := []byte{0xe9, 0xec, 0x4d, 0x9e, 2, 0, 0, 0, 3, 0, 0, 0, 0, 'k', '1', 'x', 'x', 'x'} + + assert.Equal(t, data, ent.Encode()) + + decoded := Entry{} + err := decoded.Decode(bytes.NewBuffer(data)) + assert.Nil(t, err) + assert.Equal(t, ent, decoded) + + ent = Entry{key: []byte("k1"), deleted: true} + data = []byte{0x4c, 0xd0, 0xfe, 0xe5, 2, 0, 0, 0, 0, 0, 0, 0, 1, 'k', '1'} + + assert.Equal(t, data, ent.Encode()) + + decoded = Entry{} + err = decoded.Decode(bytes.NewBuffer(data)) + assert.Nil(t, err) + assert.Equal(t, ent, decoded) +} + +// QzBQWVJJOUhU https://trialofcode.org/ diff --git a/0201/log.go b/0201/log.go new file mode 100644 index 0000000..8407820 --- /dev/null +++ b/0201/log.go @@ -0,0 +1,40 @@ +package db0201 + +import ( + "io" + "os" +) + +type Log struct { + FileName string + fp *os.File +} + +func (log *Log) Open() (err error) { + log.fp, err = createFileSync(log.FileName) + return err +} + +func (log *Log) Close() error { + return log.fp.Close() +} + +func (log *Log) Write(ent *Entry) error { + if _, err := log.fp.Write(ent.Encode()); err != nil { + return err + } + return log.fp.Sync() // fsync +} + +func (log *Log) Read(ent *Entry) (eof bool, err error) { + err = ent.Decode(log.fp) + if err == io.EOF || err == io.ErrUnexpectedEOF || err == ErrBadSum { + return true, nil + } else if err != nil { + return false, err + } else { + return false, nil + } +} + +// QzBQWVJJOUhU https://trialofcode.org/ diff --git a/0201/os_other.go b/0201/os_other.go new file mode 100644 index 0000000..6375741 --- /dev/null +++ b/0201/os_other.go @@ -0,0 +1,11 @@ +//go:build !unix + +package db0201 + +import "os" + +func createFileSync(file string) (*os.File, error) { + return os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0o644) +} + +// QzBQWVJJOUhU https://trialofcode.org/ diff --git a/0201/os_unix.go b/0201/os_unix.go new file mode 100644 index 0000000..1e2b9da --- /dev/null +++ b/0201/os_unix.go @@ -0,0 +1,34 @@ +//go:build unix + +package db0201 + +import ( + "os" + "path" + "syscall" +) + +// open or create a file and fsync the directory +func createFileSync(file string) (*os.File, error) { + fp, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0o644) + if err != nil { + return nil, err + } + if err = syncDir(path.Base(file)); err != nil { + _ = fp.Close() + return nil, err + } + return fp, err +} + +func syncDir(file string) error { + flags := os.O_RDONLY | syscall.O_DIRECTORY + dirfd, err := syscall.Open(path.Dir(file), flags, 0o644) + if err != nil { + return err + } + defer syscall.Close(dirfd) + return syscall.Fsync(dirfd) +} + +// QzBQWVJJOUhU https://trialofcode.org/