Skip to content

Commit

Permalink
implemented B-tree index (Note: more testing is needed) (#46)
Browse files Browse the repository at this point in the history 
* btree index implementation started.

* implementing bree index: WIP.

* implementing bree index: added new serialization method to Value type.

* implementing bree index: added testcase of BTreeIndex.

* implementing bree index: added codes for finalize BLTree container.

* updated bltree-go-for-embedding lib and modified testcases which uses the lib.

* implementing bree index: wrote btree index except MIN and MAX key passing. testcase has not been passed.

* implementing bree index: debugging (1).

* implementing bree index: inserting multiple records and full scan with BTreeIndex successed.

* TestKeyDuplicateInsertDeleteWithBTreeIndexInt passed!

* TestKeyDuplicateInsertDeleteWithBTreeIndex{Float, Varchar} passed!
  • Loading branch information
@ryogrid
ryogrid authored Aug 23, 2024
1 parent a495a56 commit 821fce2
Show file tree
Hide file tree
Showing 26 changed files with 595 additions and 130 deletions.
2 changes: 1 addition & 1 deletion lib/catalog/catalog_test/table_catalog_reload_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func TestTableCatalogReload(t *testing.T) {

samehada_instance_new := samehada.NewSamehadaInstance(t.Name(), common.BufferPoolMaxFrameNumForTest)
txn_new := samehada_instance_new.GetTransactionManager().Begin(nil)
catalog_recov := catalog.RecoveryCatalogFromCatalogPage(samehada_instance_new.GetBufferPoolManager(), samehada_instance_new.GetLogManager(), samehada_instance_new.GetLockManager(), txn_new)
catalog_recov := catalog.RecoveryCatalogFromCatalogPage(samehada_instance_new.GetBufferPoolManager(), samehada_instance_new.GetLogManager(), samehada_instance_new.GetLockManager(), txn_new, true)

columnToCheck := catalog_recov.GetTableByOID(1).Schema().GetColumn(1)

Expand Down
5 changes: 3 additions & 2 deletions lib/catalog/table_catalog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ func BootstrapCatalog(bpm *buffer.BufferPoolManager, log_manager *recovery.LogMa
}

// RecoveryCatalogFromCatalogPage get all information about tables and columns from disk and put it on memory
func RecoveryCatalogFromCatalogPage(bpm *buffer.BufferPoolManager, log_manager *recovery.LogManager, lock_manager *access.LockManager, txn *access.Transaction) *Catalog {
func RecoveryCatalogFromCatalogPage(bpm *buffer.BufferPoolManager, log_manager *recovery.LogManager, lock_manager *access.LockManager, txn *access.Transaction, isGracefulShutdown bool) *Catalog {
tableCatalogHeapIt := access.InitTableHeap(bpm, TableCatalogPageId, log_manager, lock_manager).Iterator(txn)

tableIds := make(map[uint32]*TableMetadata)
Expand Down Expand Up @@ -106,6 +106,7 @@ func RecoveryCatalogFromCatalogPage(bpm *buffer.BufferPoolManager, log_manager *
access.InitTableHeap(bpm, types.PageID(firstPage), log_manager, lock_manager),
uint32(oid),
log_manager,
isGracefulShutdown,
)

tableIds[uint32(oid)] = tableMetadata
Expand Down Expand Up @@ -171,7 +172,7 @@ func (c *Catalog) CreateTable(name string, schema_ *schema.Schema, txn *access.T
// attach table name as prefix to all columns name
attachTableNameToColumnsName(schema_, name_)

tableMetadata := NewTableMetadata(schema_, name_, tableHeap, oid, c.Log_manager)
tableMetadata := NewTableMetadata(schema_, name_, tableHeap, oid, c.Log_manager, true)

c.tableIdsMutex.Lock()
c.tableIds[oid] = tableMetadata
Expand Down
16 changes: 15 additions & 1 deletion lib/catalog/table_metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ type TableMetadata struct {
oid uint32
}

func NewTableMetadata(schema *schema.Schema, name string, table *access.TableHeap, oid uint32, log_manager *recovery.LogManager) *TableMetadata {
func NewTableMetadata(schema *schema.Schema, name string, table *access.TableHeap, oid uint32, log_manager *recovery.LogManager, isGracefulShutdown bool) *TableMetadata {
ret := new(TableMetadata)
ret.schema = schema
ret.name = name
Expand All @@ -42,6 +42,7 @@ func NewTableMetadata(schema *schema.Schema, name string, table *access.TableHea
// one page can store 512 key/value pair
im := index.NewIndexMetadata(column_.GetColumnName()+"_index", name, schema, []uint32{uint32(idx)})
hIdx := index.NewLinearProbeHashTableIndex(im, table.GetBufferPoolManager(), uint32(idx), common.BucketSizeOfHashIndex, column_.IndexHeaderPageId())

indexes = append(indexes, hIdx)
// at first allocation of pages for index, column's indexHeaderPageID is -1 at above code (column_.IndexHeaderPageId() == -1)
// because first allocation occurs when table creation is processed (not launched DB instace from existing db file which has difinition of this table)
Expand All @@ -63,6 +64,19 @@ func NewTableMetadata(schema *schema.Schema, name string, table *access.TableHea

indexes = append(indexes, slIdx)
//column_.SetIndexHeaderPageId(slIdx.GetHeaderPageId())
case index_constants.INDEX_KIND_BTREE:
im := index.NewIndexMetadata(column_.GetColumnName()+"_index", name, schema, []uint32{uint32(idx)})
// TODO: (SDB) need to avoid reuse of page zero when system shutdown was not graceful
var pageZeroId *int32 = nil
if column_.IndexHeaderPageId() != -1 && isGracefulShutdown {
pageZeroId = new(int32)
*pageZeroId = int32(column_.IndexHeaderPageId())
}

btrIdx := index.NewBTreeIndex(im, table.GetBufferPoolManager(), uint32(idx), log_manager, pageZeroId)

indexes = append(indexes, btrIdx)
column_.SetIndexHeaderPageId(btrIdx.GetHeaderPageId())
default:
panic("illegal index kind!")
}
Expand Down
61 changes: 61 additions & 0 deletions lib/container/btree/btree_iterator.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package btree

import (
"github.com/ryogrid/SamehadaDB/lib/samehada/samehada_util"
"github.com/ryogrid/SamehadaDB/lib/storage/buffer"
"github.com/ryogrid/SamehadaDB/lib/storage/index/index_common"
"github.com/ryogrid/SamehadaDB/lib/storage/page"
"github.com/ryogrid/SamehadaDB/lib/storage/page/skip_list_page"
"github.com/ryogrid/SamehadaDB/lib/types"
blink_tree "github.com/ryogrid/bltree-go-for-embedding"
)

type BTreeIterator struct {
bltr *blink_tree.BLTree
bpm *buffer.BufferPoolManager
curNode *skip_list_page.SkipListBlockPage
curEntry *index_common.IndexEntry
rangeStartKey *types.Value
rangeEndKey *types.Value
keyType types.TypeID
entryList []*index_common.IndexEntry
curEntryIdx int32
}

func NewSkipListIterator(bltr *blink_tree.BLTree, rangeStartKey *types.Value, rangeEndKey *types.Value) *BTreeIterator {
ret := new(BTreeIterator)

// TODO: (SDB) need to implement this
panic("Not implemented yet")
//headerPage := sl.getHeaderPage()
//
//ret.sl = sl
//ret.bpm = sl.bpm
//
//ret.rangeStartKey = rangeStartKey
//ret.rangeEndKey = rangeEndKey
//ret.keyType = headerPage.GetKeyType()
//ret.entryList = make([]*skip_list_page.IndexEntry, 0)
//
//ret.initRIDList(sl)

return ret
}

func (itr *BTreeIterator) initRIDList(bltr *blink_tree.BLTree) {
// TODO: (SDB) need to implement this
panic("Not implemented yet")
}

func (itr *BTreeIterator) Next() (done bool, err error, key *types.Value, rid *page.RID) {
// TODO: (SDB) need to implement this
panic("Not implemented yet")
if itr.curEntryIdx < int32(len(itr.entryList)) {
ret := itr.entryList[itr.curEntryIdx]
itr.curEntryIdx++
tmpRID := samehada_util.UnpackUint64toRID(ret.Value)
return false, nil, samehada_util.GetPonterOfValue(ret.Key), &tmpRID
} else {
return true, nil, nil, nil
}
}
32 changes: 16 additions & 16 deletions lib/container/btree/btree_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ func TestBLTree_deleteMany_embedding(t *testing.T) {
}

for i := range keys {
if err := bltree.InsertKey(keys[i], 0, [blink_tree.BtId]byte{0, 0, 0, 0, 0, 0}, true); err != blink_tree.BLTErrOk {
if err := bltree.InsertKey(keys[i], 0, [blink_tree.BtId]byte{0, 0, 0, 0, 0, 0, 0, 0}, true); err != blink_tree.BLTErrOk {
t.Errorf("InsertKey() = %v, want %v", err, blink_tree.BLTErrOk)
}
if i%2 == 0 {
Expand All @@ -128,8 +128,8 @@ func TestBLTree_deleteMany_embedding(t *testing.T) {
t.Errorf("FindKey() = %v, want %v, key %v", found, -1, keys[i])
}
} else {
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i])
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i])
}
}
}
Expand All @@ -154,7 +154,7 @@ func TestBLTree_deleteAll_embedding(t *testing.T) {
}

for i := range keys {
if err := bltree.InsertKey(keys[i], 0, [blink_tree.BtId]byte{0, 0, 0, 0, 0, 0}, true); err != blink_tree.BLTErrOk {
if err := bltree.InsertKey(keys[i], 0, [blink_tree.BtId]byte{0, 0, 0, 0, 0, 0, 0, 0}, true); err != blink_tree.BLTErrOk {
t.Errorf("InsertKey() = %v, want %v", err, blink_tree.BLTErrOk)
}
}
Expand Down Expand Up @@ -216,8 +216,8 @@ func TestBLTree_deleteManyConcurrently_embedding(t *testing.T) {
panic("FindKey() != -1")
}
} else {
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i])
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i])
panic("FindKey() != 6")
}
}
Expand Down Expand Up @@ -245,8 +245,8 @@ func TestBLTree_deleteManyConcurrently_embedding(t *testing.T) {
t.Errorf("FindKey() = %v, want %v, key %v", found, -1, keys[i])
}
} else {
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i])
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i])
}
}
}
Expand Down Expand Up @@ -330,8 +330,8 @@ func TestBLTree_deleteInsertRangeScanConcurrently_embedding(t *testing.T) {
}
rangeScanCheck(keys[i])
} else {
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i])
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i])
panic("FindKey() != 6")
}
rangeScanCheck(keys[i])
Expand Down Expand Up @@ -363,8 +363,8 @@ func TestBLTree_deleteInsertRangeScanConcurrently_embedding(t *testing.T) {
}
}
} else {
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i])
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i])
}
}
}
Expand Down Expand Up @@ -428,8 +428,8 @@ func TestBLTree_deleteManyConcurrentlyShuffle_embedding(t *testing.T) {
panic("FindKey() != -1")
}
} else {
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i])
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i])
panic("FindKey() != 6")
}
}
Expand Down Expand Up @@ -457,8 +457,8 @@ func TestBLTree_deleteManyConcurrentlyShuffle_embedding(t *testing.T) {
t.Errorf("FindKey() = %v, want %v, key %v", found, -1, keys[i])
}
} else {
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i])
if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 {
t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i])
}
}
}
Expand Down
7 changes: 4 additions & 3 deletions lib/container/skip_list/skip_list_iterator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package skip_list
import (
"github.com/ryogrid/SamehadaDB/lib/samehada/samehada_util"
"github.com/ryogrid/SamehadaDB/lib/storage/buffer"
"github.com/ryogrid/SamehadaDB/lib/storage/index/index_common"
"github.com/ryogrid/SamehadaDB/lib/storage/page"
"github.com/ryogrid/SamehadaDB/lib/storage/page/skip_list_page"
"github.com/ryogrid/SamehadaDB/lib/types"
Expand All @@ -12,11 +13,11 @@ type SkipListIterator struct {
sl *SkipList
bpm *buffer.BufferPoolManager
curNode *skip_list_page.SkipListBlockPage
curEntry *skip_list_page.SkipListPair
curEntry *index_common.IndexEntry
rangeStartKey *types.Value
rangeEndKey *types.Value
keyType types.TypeID
entryList []*skip_list_page.SkipListPair
entryList []*index_common.IndexEntry
curEntryIdx int32
}

Expand All @@ -31,7 +32,7 @@ func NewSkipListIterator(sl *SkipList, rangeStartKey *types.Value, rangeEndKey *
ret.rangeStartKey = rangeStartKey
ret.rangeEndKey = rangeEndKey
ret.keyType = headerPage.GetKeyType()
ret.entryList = make([]*skip_list_page.SkipListPair, 0)
ret.entryList = make([]*index_common.IndexEntry, 0)

ret.initRIDList(sl)

Expand Down
Loading

0 comments on commit 821fce2

Please sign in to comment.