Skip to content

Commit

Permalink
updated bltree-go-for-embedding lib and modified testcases which uses…
Browse files Browse the repository at this point in the history 
… the lib.
  • Loading branch information
@ryogrid
ryogrid committed Aug 23, 2024
1 parent 8a22240 commit 2966f4f
Show file tree
Hide file tree
Showing 9 changed files with 40 additions and 81 deletions.
41 changes: 0 additions & 41 deletions lib/container/btree/bltree_wrapper.go
View file

This file was deleted.

7 changes: 4 additions & 3 deletions lib/container/btree/btree_iterator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@ import (
"github.com/ryogrid/SamehadaDB/lib/storage/page"
"github.com/ryogrid/SamehadaDB/lib/storage/page/skip_list_page"
"github.com/ryogrid/SamehadaDB/lib/types"
blink_tree "github.com/ryogrid/bltree-go-for-embedding"
)

type BTreeIterator struct {
bltw *BLTreeWrapper
bltr *blink_tree.BLTree
bpm *buffer.BufferPoolManager
curNode *skip_list_page.SkipListBlockPage
curEntry *index_common.IndexEntry
Expand All @@ -21,7 +22,7 @@ type BTreeIterator struct {
curEntryIdx int32
}

func NewSkipListIterator(bltr *BLTreeWrapper, rangeStartKey *types.Value, rangeEndKey *types.Value) *BTreeIterator {
func NewSkipListIterator(bltr *blink_tree.BLTree, rangeStartKey *types.Value, rangeEndKey *types.Value) *BTreeIterator {
ret := new(BTreeIterator)

// TODO: (SDB) need to implement this
Expand All @@ -41,7 +42,7 @@ func NewSkipListIterator(bltr *BLTreeWrapper, rangeStartKey *types.Value, rangeE
return ret
}

func (itr *BTreeIterator) initRIDList(bltw *BLTreeWrapper) {
func (itr *BTreeIterator) initRIDList(bltr *blink_tree.BLTree) {
// TODO: (SDB) need to implement this
panic("Not implemented yet")
}
Expand Down
32 changes: 16 additions & 16 deletions lib/container/btree/btree_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ func TestBLTree_deleteMany_embedding(t *testing.T) {
}

for i := range keys {
if err := bltree.InsertKey(keys[i], 0, [blink_tree.BtId]byte{0, 0, 0, 0, 0, 0}, true); err != blink_tree.BLTErrOk {
if err := bltree.InsertKey(keys[i], 0, [blink_tree.BtId]byte{0, 0, 0, 0, 0, 0, 0, 0}, true); err != blink_tree.BLTErrOk {
t.Errorf("InsertKey() = %v, want %v", err, blink_tree.BLTErrOk)
}
if i%2 == 0 {
Expand All @@ -128,8 +128,8 @@ func TestBLTree_deleteMany_embedding(t *testing.T) {
t.Errorf("FindKey() = %v, want %v, key %v", found, -1, keys[i])
}
} else {
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i])
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i])
}
}
}
Expand All @@ -154,7 +154,7 @@ func TestBLTree_deleteAll_embedding(t *testing.T) {
}

for i := range keys {
if err := bltree.InsertKey(keys[i], 0, [blink_tree.BtId]byte{0, 0, 0, 0, 0, 0}, true); err != blink_tree.BLTErrOk {
if err := bltree.InsertKey(keys[i], 0, [blink_tree.BtId]byte{0, 0, 0, 0, 0, 0, 0, 0}, true); err != blink_tree.BLTErrOk {
t.Errorf("InsertKey() = %v, want %v", err, blink_tree.BLTErrOk)
}
}
Expand Down Expand Up @@ -216,8 +216,8 @@ func TestBLTree_deleteManyConcurrently_embedding(t *testing.T) {
panic("FindKey() != -1")
}
} else {
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i])
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i])
panic("FindKey() != 6")
}
}
Expand Down Expand Up @@ -245,8 +245,8 @@ func TestBLTree_deleteManyConcurrently_embedding(t *testing.T) {
t.Errorf("FindKey() = %v, want %v, key %v", found, -1, keys[i])
}
} else {
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i])
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i])
}
}
}
Expand Down Expand Up @@ -330,8 +330,8 @@ func TestBLTree_deleteInsertRangeScanConcurrently_embedding(t *testing.T) {
}
rangeScanCheck(keys[i])
} else {
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i])
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i])
panic("FindKey() != 6")
}
rangeScanCheck(keys[i])
Expand Down Expand Up @@ -363,8 +363,8 @@ func TestBLTree_deleteInsertRangeScanConcurrently_embedding(t *testing.T) {
}
}
} else {
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i])
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i])
}
}
}
Expand Down Expand Up @@ -428,8 +428,8 @@ func TestBLTree_deleteManyConcurrentlyShuffle_embedding(t *testing.T) {
panic("FindKey() != -1")
}
} else {
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i])
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i])
panic("FindKey() != 6")
}
}
Expand Down Expand Up @@ -457,8 +457,8 @@ func TestBLTree_deleteManyConcurrentlyShuffle_embedding(t *testing.T) {
t.Errorf("FindKey() = %v, want %v, key %v", found, -1, keys[i])
}
} else {
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i])
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i])
}
}
}
Expand Down
1 change: 1 addition & 0 deletions lib/execution/executors/executor_test/btree_index_executor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ func testKeyDuplicateInsertDeleteWithBTreeIndex[T float32 | int32 | string](t *t
testingpkg.Assert(t, len(result) == 0, "duplicated key point scan got illegal results.")

txnMgr.Commit(c, txn)
shi.Shutdown(samehada.ShutdownPatternCloseFiles)
}

func TestKeyDuplicateInsertDeleteWithBTreeIndexInt(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion lib/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ require (
github.com/notEpsilon/go-pair v0.0.0-20221220200415-e91ef28c6c0b
github.com/pingcap/parser v0.0.0-20200623164729-3a18f1e5dceb
github.com/pingcap/tidb v1.1.0-beta.0.20200630082100-328b6d0a955c
github.com/ryogrid/bltree-go-for-embedding v1.0.2
github.com/ryogrid/bltree-go-for-embedding v1.0.3
github.com/spaolacci/murmur3 v1.1.0
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
)
Expand Down
4 changes: 2 additions & 2 deletions lib/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -448,8 +448,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryogrid/bltree-go-for-embedding v1.0.2 h1:GquFei9pmPy4GK9YBGwXITcJFCnRjWTX+BDpaIVxJLI=
github.com/ryogrid/bltree-go-for-embedding v1.0.2/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE=
github.com/ryogrid/bltree-go-for-embedding v1.0.3 h1:EvhpPa8UeD+BD/hM46qbqbWCQMLRAqHE0A4l/hywamU=
github.com/ryogrid/bltree-go-for-embedding v1.0.3/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE=
github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
Expand Down
28 changes: 13 additions & 15 deletions lib/storage/index/btree_index.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,27 +16,30 @@ import (
)

type BTreeIndex struct {
container *btree.BLTreeWrapper
container *blink_tree.BLTree
metadata *IndexMetadata
// idx of target column on table
col_idx uint32
log_manager *recovery.LogManager
// UpdateEntry only get Write lock
updateMtx sync.RWMutex
// for call of Close method ....
bufMgr *blink_tree.BufMgr
}

func NewBTreeIndex(metadata *IndexMetadata, buffer_pool_manager *buffer.BufferPoolManager, col_idx uint32, log_manager *recovery.LogManager) *BTreeIndex {
func NewBTreeIndex(metadata *IndexMetadata, buffer_pool_manager *buffer.BufferPoolManager, col_idx uint32, log_manager *recovery.LogManager, lastPageZeroId *int32) *BTreeIndex {
ret := new(BTreeIndex)
ret.metadata = metadata

// BTreeIndex uses special technique to support key duplication with SkipList supporting unique key only
// for the thechnique, key type is fixed to Varchar (comparison is done on dict order as byte array)

bufMgr := blink_tree.NewBufMgr(12, blink_tree.HASH_TABLE_ENTRY_CHAIN_LEN*common.MaxTxnThreadNum*2, btree.NewParentBufMgrImpl(buffer_pool_manager), nil)
ret.container = btree.NewBLTreeWrapper(blink_tree.NewBLTree(bufMgr), bufMgr)
bufMgr := blink_tree.NewBufMgr(12, blink_tree.HASH_TABLE_ENTRY_CHAIN_LEN*common.MaxTxnThreadNum*2, btree.NewParentBufMgrImpl(buffer_pool_manager), lastPageZeroId)
ret.container = blink_tree.NewBLTree(bufMgr)
ret.col_idx = col_idx
ret.updateMtx = sync.RWMutex{}
ret.log_manager = log_manager
ret.bufMgr = bufMgr
return ret
}

Expand All @@ -63,11 +66,6 @@ func (btidx *BTreeIndex) deleteEntryInner(key *tuple.Tuple, rid page.RID, txn in

convedKeyVal := samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgKeyVal, &rid)

//revertedOrgKey := samehada_util.ExtractOrgKeyFromDicOrderComparableEncodedVarchar(convedKeyVal, orgKeyVal.ValueType())
//if !revertedOrgKey.CompareEquals(orgKeyVal) {
// panic("key conversion may fail!")
//}

if isNoLock == false {
btidx.updateMtx.RLock()
defer btidx.updateMtx.RUnlock()
Expand Down Expand Up @@ -146,12 +144,12 @@ func (btidx *BTreeIndex) GetTupleSchema() *schema.Schema {

func (btidx *BTreeIndex) GetKeyAttrs() []uint32 { return btidx.metadata.GetKeyAttrs() }

//func (slidx *BTreeIndex) GetHeaderPageId() types.PageID {
// return slidx.container.GetHeaderPageId()
//}
func (slidx *BTreeIndex) GetHeaderPageId() types.PageID {
return types.PageID(slidx.bufMgr.GetMappedPPageIdOfPageZero())
}

// call this at shutdown of the system
// to write out the state and allocated pages of the container to BPM
func (btidx *BTreeIndex) Close() {
btidx.container.WriteOutContainerStateToBPM()
// to write out the state and allocated pages of the BLTree container to BPM
func (btidx *BTreeIndex) WriteOutContainerStateToBPM() {
btidx.bufMgr.Close()
}
2 changes: 1 addition & 1 deletion server/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ require (
github.com/pingcap/tidb v1.1.0-beta.0.20200630082100-328b6d0a955c // indirect
github.com/pingcap/tipb v0.0.0-20200522051215-f31a15d98fce // indirect
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
github.com/ryogrid/bltree-go-for-embedding v1.0.2 // indirect
github.com/ryogrid/bltree-go-for-embedding v1.0.3 // indirect
github.com/shirou/gopsutil v2.19.10+incompatible // indirect
github.com/sirupsen/logrus v1.6.0 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions server/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -450,8 +450,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryogrid/bltree-go-for-embedding v1.0.2 h1:GquFei9pmPy4GK9YBGwXITcJFCnRjWTX+BDpaIVxJLI=
github.com/ryogrid/bltree-go-for-embedding v1.0.2/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE=
github.com/ryogrid/bltree-go-for-embedding v1.0.3 h1:EvhpPa8UeD+BD/hM46qbqbWCQMLRAqHE0A4l/hywamU=
github.com/ryogrid/bltree-go-for-embedding v1.0.3/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE=
github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
Expand Down

0 comments on commit 2966f4f

Please sign in to comment.