Skip to content

Commit

Permalink
Merge pull request #15 from octu0/v1.5.0
Browse files Browse the repository at this point in the history 
v1.5.0
  • Loading branch information
@octu0
octu0 authored Jul 14, 2022
2 parents 5cc4dee + 8b23bb6 commit b391ba1
Show file tree
Hide file tree
Showing 5 changed files with 42 additions and 51 deletions.
29 changes: 8 additions & 21 deletions bitcask.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"bytes"
"encoding/json"
"io"
"math"
"os"
"path/filepath"
"sort"
Expand All @@ -13,9 +12,9 @@ import (

"github.com/abcum/lcp"
"github.com/gofrs/flock"
"github.com/octu0/priorate"
"github.com/pkg/errors"
art "github.com/plar/go-adaptive-radix-tree"
"golang.org/x/time/rate"
"github.com/plar/go-adaptive-radix-tree"

"github.com/octu0/bitcaskdb/datafile"
"github.com/octu0/bitcaskdb/indexer"
Expand Down Expand Up @@ -718,15 +717,15 @@ func (b *Bitcask) reopenLocked() error {
// and deleted keys removes. Duplicate key/value pairs are also removed.
// Call this function periodically to reclaim disk space.
func (b *Bitcask) Merge() error {
return b.merger.Merge(b, rate.NewLimiter(rate.Inf, math.MaxInt))
return b.merger.Merge(b, priorate.InfLimiter())
}

func (b *Bitcask) MergeWithWaitLimit(lim *rate.Limiter) error {
func (b *Bitcask) MergeWithWaitLimit(lim *priorate.Limiter) error {
return b.merger.Merge(b, lim)
}

func (b *Bitcask) MergeWithWaitLimitByBytesPerSecond(bytesPerSecond int) error {
return b.merger.Merge(b, rate.NewLimiter(rate.Limit(float64(bytesPerSecond)), bytesPerSecond))
return b.merger.Merge(b, priorate.NewLimiter(bytesPerSecond))
}

// saveIndex saves index and ttl_index currently in RAM to disk
Expand Down Expand Up @@ -821,7 +820,7 @@ func loadIndexes(b *Bitcask, datafiles map[datafile.FileID]datafile.Datafile, la
return t, ttlIndex, nil
}
if found {
if err := loadIndexFromDatafile(t, ttlIndex, datafiles[lastID], nil); err != nil {
if err := loadIndexFromDatafile(t, ttlIndex, datafiles[lastID]); err != nil {
return nil, ttlIndex, errors.WithStack(err)
}
return t, ttlIndex, nil
Expand All @@ -837,18 +836,14 @@ func loadIndexes(b *Bitcask, datafiles map[datafile.FileID]datafile.Datafile, la

for _, fileID := range fileIds {
df := datafiles[fileID]
if err := loadIndexFromDatafile(t, ttlIndex, df, nil); err != nil {
if err := loadIndexFromDatafile(t, ttlIndex, df); err != nil {
return nil, ttlIndex, errors.WithStack(err)
}
}
return t, ttlIndex, nil
}

func loadIndexFromDatafile(t art.Tree, ttlIndex art.Tree, df datafile.Datafile, lim *rate.Limiter) error {
if lim == nil {
lim = rate.NewLimiter(rate.Inf, math.MaxInt)
}

func loadIndexFromDatafile(t art.Tree, ttlIndex art.Tree, df datafile.Datafile) error {
index := int64(0)
for {
e, err := df.Read()
Expand All @@ -875,14 +870,6 @@ func loadIndexFromDatafile(t art.Tree, ttlIndex art.Tree, df datafile.Datafile,
t.Delete(e.Key)
index += e.TotalSize
}

r := lim.ReserveN(time.Now(), int(e.TotalSize))
if r.OK() != true {
continue
}
if d := r.Delay(); 0 < d {
time.Sleep(d)
}
}
return nil
}
Expand Down
3 changes: 2 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ require (
github.com/nats-io/nats-server/v2 v2.8.4
github.com/nats-io/nats.go v1.16.0
github.com/octu0/bp v1.2.0
github.com/octu0/priorate v1.0.0
github.com/pkg/errors v0.9.1
github.com/plar/go-adaptive-radix-tree v1.0.4
github.com/stretchr/testify v1.7.0
golang.org/x/exp v0.0.0-20200228211341-fcea875c7e85
golang.org/x/time v0.0.0-20220609170525-579cf78fd858
)

require (
Expand All @@ -27,5 +27,6 @@ require (
github.com/sirupsen/logrus v1.8.1 // indirect
golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect
golang.org/x/sys v0.0.0-20220111092808-5a964db01320 // indirect
golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)
2 changes: 2 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,8 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS
github.com/octu0/bp v1.2.0 h1:WKvXhEpwFOyi6Nszbrlfng9hEViSx6AptFeY37zDsKY=
github.com/octu0/bp v1.2.0/go.mod h1:e9Mr74DWvtMAKYr5HD121M+FB5WBcI8yBP+b5Lu4qv8=
github.com/octu0/chanque v1.0.15 h1:5KeAqdT6GcHgtS0mjnXIuye58vuexl+DbjE1VW0rm64=
github.com/octu0/priorate v1.0.0 h1:7fX4q1kzgulZ5W2UiTurfIqtZ/bnVgdlPzyi6bnBW7w=
github.com/octu0/priorate v1.0.0/go.mod h1:zrh79ghtyVMsD1C1QAJeOiM/lID6eaWHjAjk7mNkaP8=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
Expand Down
57 changes: 29 additions & 28 deletions merge.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ import (
"sync"
"time"

"github.com/octu0/priorate"
"github.com/pkg/errors"
art "github.com/plar/go-adaptive-radix-tree"
"golang.org/x/time/rate"
"github.com/plar/go-adaptive-radix-tree"

"github.com/octu0/bitcaskdb/datafile"
"github.com/octu0/bitcaskdb/indexer"
Expand All @@ -27,7 +27,8 @@ const (
)

const (
defaultTruncateThreshold int64 = 100 * 1024 * 1024
defaultTruncateThreshold int64 = 100 * 1024 * 1024
defaultSlowTruncateWait time.Duration = 100 * time.Millisecond
)

type merger struct {
Expand All @@ -42,7 +43,7 @@ func (m *merger) isMerging() bool {
return m.merging
}

func (m *merger) Merge(b *Bitcask, lim *rate.Limiter) error {
func (m *merger) Merge(b *Bitcask, lim *priorate.Limiter) error {
if m.isMerging() {
return errors.WithStack(ErrMergeInProgress)
}
Expand Down Expand Up @@ -84,23 +85,23 @@ func (m *merger) Merge(b *Bitcask, lim *rate.Limiter) error {
return nil
}

func (m *merger) tellSaveIndexCostLocked(b *Bitcask, lim *rate.Limiter) {
func (m *merger) tellSaveIndexCostLocked(b *Bitcask, lim *priorate.Limiter) {
saveCostFiler := b.trie.Size() * indexer.FilerByteSize
saveCostTTL := b.ttlIndex.Size() * 8

lim.ReserveN(time.Now(), saveCostFiler)
lim.ReserveN(time.Now(), saveCostTTL)
lim.ReserveN(priorate.High, time.Now(), saveCostFiler)
lim.ReserveN(priorate.High, time.Now(), saveCostTTL)
}

func (m *merger) tellLoadIndexCostLocked(b *Bitcask, lim *rate.Limiter) {
func (m *merger) tellLoadIndexCostLocked(b *Bitcask, lim *priorate.Limiter) {
loadCostFiler := b.trie.Size() * indexer.FilerByteSize
loadCostTTL := b.ttlIndex.Size() * 8

lim.ReserveN(time.Now(), loadCostFiler)
lim.ReserveN(time.Now(), loadCostTTL)
lim.ReserveN(priorate.Low, time.Now(), loadCostFiler)
lim.ReserveN(priorate.Low, time.Now(), loadCostTTL)
}

func (m *merger) reopen(b *Bitcask, temp *mergeTempDB, lastFileID datafile.FileID, lim *rate.Limiter) ([]string, error) {
func (m *merger) reopen(b *Bitcask, temp *mergeTempDB, lastFileID datafile.FileID, lim *priorate.Limiter) ([]string, error) {
// no reads and writes till we reopen
b.mu.Lock()
defer b.mu.Unlock()
Expand Down Expand Up @@ -155,7 +156,7 @@ func (m *merger) forwardCurrentDafafile(b *Bitcask) (datafile.FileID, []datafile
return currentFileID, mergeFileIds, nil
}

func (m *merger) snapshotIndexer(b *Bitcask, lim *rate.Limiter) (*snapshotTrie, error) {
func (m *merger) snapshotIndexer(b *Bitcask, lim *priorate.Limiter) (*snapshotTrie, error) {
b.mu.RLock()
defer b.mu.RUnlock()

Expand All @@ -166,7 +167,7 @@ func (m *merger) snapshotIndexer(b *Bitcask, lim *rate.Limiter) (*snapshotTrie,

var lastErr error
b.trie.ForEach(func(node art.Node) bool {
r := lim.ReserveN(time.Now(), indexer.FilerByteSize)
r := lim.ReserveN(priorate.Low, time.Now(), indexer.FilerByteSize)
if r.OK() {
if d := r.Delay(); 0 < d {
time.Sleep(d)
Expand All @@ -185,7 +186,7 @@ func (m *merger) snapshotIndexer(b *Bitcask, lim *rate.Limiter) (*snapshotTrie,
return st, nil
}

func (m *merger) renewMergedDB(b *Bitcask, mergeFileIds []datafile.FileID, st *snapshotTrie, lim *rate.Limiter) (*mergeTempDB, error) {
func (m *merger) renewMergedDB(b *Bitcask, mergeFileIds []datafile.FileID, st *snapshotTrie, lim *priorate.Limiter) (*mergeTempDB, error) {
temp, err := openMergeTempDB(b.path, b.opt)
if err != nil {
return nil, errors.WithStack(err)
Expand Down Expand Up @@ -299,7 +300,7 @@ func (t *mergeTempDB) DB() *Bitcask {
// Rewrite all key/value pairs into merged database
// Doing this automatically strips deleted keys and
// old key/value pairs
func (t *mergeTempDB) MergeDatafiles(src *Bitcask, mergeFileIds []datafile.FileID, st *snapshotTrie, lim *rate.Limiter) error {
func (t *mergeTempDB) MergeDatafiles(src *Bitcask, mergeFileIds []datafile.FileID, st *snapshotTrie, lim *priorate.Limiter) error {
t.mdb.mu.Lock()
defer t.mdb.mu.Unlock()

Expand Down Expand Up @@ -327,7 +328,7 @@ func (t *mergeTempDB) MergeDatafiles(src *Bitcask, mergeFileIds []datafile.FileI
return nil
}

func (t *mergeTempDB) mergeDatafileLocked(st *snapshotTrie, m map[datafile.FileID]datafile.Datafile, lim *rate.Limiter) error {
func (t *mergeTempDB) mergeDatafileLocked(st *snapshotTrie, m map[datafile.FileID]datafile.Datafile, lim *priorate.Limiter) error {
return st.ReadAll(func(data snapshotTrieData) error {
filer := data.Value

Expand All @@ -336,7 +337,7 @@ func (t *mergeTempDB) mergeDatafileLocked(st *snapshotTrie, m map[datafile.FileI
return nil
}

rr := lim.ReserveN(time.Now(), int(filer.Size))
rr := lim.ReserveN(priorate.Low, time.Now(), int(filer.Size))
if rr.OK() {
if d := rr.Delay(); 0 < d {
time.Sleep(d)
Expand All @@ -354,7 +355,7 @@ func (t *mergeTempDB) mergeDatafileLocked(st *snapshotTrie, m map[datafile.FileI
return nil
}

rw := lim.ReserveN(time.Now(), int(e.TotalSize))
rw := lim.ReserveN(priorate.Low, time.Now(), int(e.TotalSize))
if rw.OK() {
if d := rw.Delay(); 0 < d {
time.Sleep(d)
Expand Down Expand Up @@ -383,7 +384,7 @@ func (t *mergeTempDB) SyncAndClose() error {
return nil
}

func (t *mergeTempDB) Destroy(lim *rate.Limiter) {
func (t *mergeTempDB) Destroy(lim *priorate.Limiter) {
if t.destroyed {
return
}
Expand Down Expand Up @@ -472,7 +473,7 @@ func (st *snapshotTrie) ReadAll(fn func(snapshotTrieData) error) error {
return nil
}

func (st *snapshotTrie) Destroy(lim *rate.Limiter) {
func (st *snapshotTrie) Destroy(lim *priorate.Limiter) {
if st.destroyed {
return
}
Expand Down Expand Up @@ -502,9 +503,9 @@ func openSnapshotTrie(tempDir string) (*snapshotTrie, error) {
return st, nil
}

func removeFileSlowly(files []string, lim *rate.Limiter) error {
func removeFileSlowly(files []string, lim *priorate.Limiter) error {
if lim == nil {
lim = rate.NewLimiter(rate.Inf, math.MaxInt)
lim = priorate.InfLimiter()
}

for _, file := range files {
Expand All @@ -517,7 +518,7 @@ func removeFileSlowly(files []string, lim *rate.Limiter) error {
}
filesize := stat.Size()

if defaultTruncateThreshold < filesize {
if lim.Limit() < math.MaxInt && int64(lim.Limit()) < filesize {
truncate(file, filesize, lim)
}
if err := os.Remove(file); err != nil {
Expand All @@ -527,20 +528,20 @@ func removeFileSlowly(files []string, lim *rate.Limiter) error {
return nil
}

func truncate(path string, size int64, lim *rate.Limiter) {
func truncate(path string, size int64, lim *priorate.Limiter) {
threshold := int64(defaultTruncateThreshold)
if lim.Limit() < rate.Inf && lim.Limit() < math.MaxInt {
if lim.Limit() < math.MaxInt {
threshold = int64(lim.Limit())
}

truncateCount := (size - 1) / threshold
for i := int64(0); i < truncateCount; i += 1 {
nextSize := defaultTruncateThreshold * (truncateCount - i)
nextSize := threshold * (truncateCount - i)

r := lim.ReserveN(time.Now(), int(nextSize))
r := lim.ReserveN(priorate.Low, time.Now(), int(nextSize))
if r.OK() {
if d := r.Delay(); 0 < d {
time.Sleep(d)
time.Sleep(d + defaultSlowTruncateWait)
}
}

Expand Down
2 changes: 1 addition & 1 deletion version.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@ package bitcaskdb

const (
AppName string = "bitcaskdb"
Version string = "1.4.1"
Version string = "1.5.0"
)

0 comments on commit b391ba1

Please sign in to comment.