From 821fce25d5649e463b814e07211c687e72ae6653 Mon Sep 17 00:00:00 2001 From: Ryo Kanbayashi Date: Fri, 23 Aug 2024 22:03:16 +0900 Subject: [PATCH] implemented B-tree index (Note: more testing is needed) (#46) * btree index implementation started. * implementing bree index: WIP. * implementing bree index: added new serialization method to Value type. * implementing bree index: added testcase of BTreeIndex. * implementing bree index: added codes for finalize BLTree container. * updated bltree-go-for-embedding lib and modified testcases which uses the lib. * implementing bree index: wrote btree index except MIN and MAX key passing. testcase has not been passed. * implementing bree index: debugging (1). * implementing bree index: inserting multiple records and full scan with BTreeIndex successed. * TestKeyDuplicateInsertDeleteWithBTreeIndexInt passed! * TestKeyDuplicateInsertDeleteWithBTreeIndex{Float, Varchar} passed! --- .../catalog_test/table_catalog_reload_test.go | 2 +- lib/catalog/table_catalog.go | 5 +- lib/catalog/table_metadata.go | 16 +- lib/container/btree/btree_iterator.go | 61 ++++++ lib/container/btree/btree_test.go | 32 +-- lib/container/skip_list/skip_list_iterator.go | 7 +- .../skip_list_test/skip_list_test.go | 55 ++--- .../btree_index_executor_test.go | 122 +++++++++++ .../executors/executor_test/executor_test.go | 8 +- .../skiplist_index_executor_test.go | 20 +- .../uniq_skiplist_index_executor_test.go | 4 +- .../range_scan_with_index_executor.go | 12 ++ lib/go.mod | 2 +- lib/go.sum | 4 +- lib/samehada/samehada.go | 6 +- lib/samehada/samehada_instance.go | 1 + lib/samehada/samehada_util/samehada_util.go | 18 +- lib/storage/index/btree_index.go | 197 ++++++++++++++++++ .../index/index_common/index_common.go | 40 ++++ .../index/index_constants/index_constants.go | 1 + .../index/index_test/hash_table_index_test.go | 2 +- .../skip_list_page/skip_list_block_page.go | 67 ++---- .../skip_list_page/skip_list_header_page.go | 17 +- lib/types/column_value.go | 21 ++ server/go.mod | 3 +- server/go.sum | 2 + 26 files changed, 595 insertions(+), 130 deletions(-) create mode 100644 lib/container/btree/btree_iterator.go create mode 100644 lib/execution/executors/executor_test/btree_index_executor_test.go create mode 100644 lib/storage/index/btree_index.go create mode 100644 lib/storage/index/index_common/index_common.go diff --git a/lib/catalog/catalog_test/table_catalog_reload_test.go b/lib/catalog/catalog_test/table_catalog_reload_test.go index 957923f2..efe81eef 100644 --- a/lib/catalog/catalog_test/table_catalog_reload_test.go +++ b/lib/catalog/catalog_test/table_catalog_reload_test.go @@ -45,7 +45,7 @@ func TestTableCatalogReload(t *testing.T) { samehada_instance_new := samehada.NewSamehadaInstance(t.Name(), common.BufferPoolMaxFrameNumForTest) txn_new := samehada_instance_new.GetTransactionManager().Begin(nil) - catalog_recov := catalog.RecoveryCatalogFromCatalogPage(samehada_instance_new.GetBufferPoolManager(), samehada_instance_new.GetLogManager(), samehada_instance_new.GetLockManager(), txn_new) + catalog_recov := catalog.RecoveryCatalogFromCatalogPage(samehada_instance_new.GetBufferPoolManager(), samehada_instance_new.GetLogManager(), samehada_instance_new.GetLockManager(), txn_new, true) columnToCheck := catalog_recov.GetTableByOID(1).Schema().GetColumn(1) diff --git a/lib/catalog/table_catalog.go b/lib/catalog/table_catalog.go index 1a7bc0fe..4fb5bdb1 100644 --- a/lib/catalog/table_catalog.go +++ b/lib/catalog/table_catalog.go @@ -62,7 +62,7 @@ func BootstrapCatalog(bpm *buffer.BufferPoolManager, log_manager *recovery.LogMa } // RecoveryCatalogFromCatalogPage get all information about tables and columns from disk and put it on memory -func RecoveryCatalogFromCatalogPage(bpm *buffer.BufferPoolManager, log_manager *recovery.LogManager, lock_manager *access.LockManager, txn *access.Transaction) *Catalog { +func RecoveryCatalogFromCatalogPage(bpm *buffer.BufferPoolManager, log_manager *recovery.LogManager, lock_manager *access.LockManager, txn *access.Transaction, isGracefulShutdown bool) *Catalog { tableCatalogHeapIt := access.InitTableHeap(bpm, TableCatalogPageId, log_manager, lock_manager).Iterator(txn) tableIds := make(map[uint32]*TableMetadata) @@ -106,6 +106,7 @@ func RecoveryCatalogFromCatalogPage(bpm *buffer.BufferPoolManager, log_manager * access.InitTableHeap(bpm, types.PageID(firstPage), log_manager, lock_manager), uint32(oid), log_manager, + isGracefulShutdown, ) tableIds[uint32(oid)] = tableMetadata @@ -171,7 +172,7 @@ func (c *Catalog) CreateTable(name string, schema_ *schema.Schema, txn *access.T // attach table name as prefix to all columns name attachTableNameToColumnsName(schema_, name_) - tableMetadata := NewTableMetadata(schema_, name_, tableHeap, oid, c.Log_manager) + tableMetadata := NewTableMetadata(schema_, name_, tableHeap, oid, c.Log_manager, true) c.tableIdsMutex.Lock() c.tableIds[oid] = tableMetadata diff --git a/lib/catalog/table_metadata.go b/lib/catalog/table_metadata.go index afdb6dff..08d6bdf4 100644 --- a/lib/catalog/table_metadata.go +++ b/lib/catalog/table_metadata.go @@ -24,7 +24,7 @@ type TableMetadata struct { oid uint32 } -func NewTableMetadata(schema *schema.Schema, name string, table *access.TableHeap, oid uint32, log_manager *recovery.LogManager) *TableMetadata { +func NewTableMetadata(schema *schema.Schema, name string, table *access.TableHeap, oid uint32, log_manager *recovery.LogManager, isGracefulShutdown bool) *TableMetadata { ret := new(TableMetadata) ret.schema = schema ret.name = name @@ -42,6 +42,7 @@ func NewTableMetadata(schema *schema.Schema, name string, table *access.TableHea // one page can store 512 key/value pair im := index.NewIndexMetadata(column_.GetColumnName()+"_index", name, schema, []uint32{uint32(idx)}) hIdx := index.NewLinearProbeHashTableIndex(im, table.GetBufferPoolManager(), uint32(idx), common.BucketSizeOfHashIndex, column_.IndexHeaderPageId()) + indexes = append(indexes, hIdx) // at first allocation of pages for index, column's indexHeaderPageID is -1 at above code (column_.IndexHeaderPageId() == -1) // because first allocation occurs when table creation is processed (not launched DB instace from existing db file which has difinition of this table) @@ -63,6 +64,19 @@ func NewTableMetadata(schema *schema.Schema, name string, table *access.TableHea indexes = append(indexes, slIdx) //column_.SetIndexHeaderPageId(slIdx.GetHeaderPageId()) + case index_constants.INDEX_KIND_BTREE: + im := index.NewIndexMetadata(column_.GetColumnName()+"_index", name, schema, []uint32{uint32(idx)}) + // TODO: (SDB) need to avoid reuse of page zero when system shutdown was not graceful + var pageZeroId *int32 = nil + if column_.IndexHeaderPageId() != -1 && isGracefulShutdown { + pageZeroId = new(int32) + *pageZeroId = int32(column_.IndexHeaderPageId()) + } + + btrIdx := index.NewBTreeIndex(im, table.GetBufferPoolManager(), uint32(idx), log_manager, pageZeroId) + + indexes = append(indexes, btrIdx) + column_.SetIndexHeaderPageId(btrIdx.GetHeaderPageId()) default: panic("illegal index kind!") } diff --git a/lib/container/btree/btree_iterator.go b/lib/container/btree/btree_iterator.go new file mode 100644 index 00000000..13840682 --- /dev/null +++ b/lib/container/btree/btree_iterator.go @@ -0,0 +1,61 @@ +package btree + +import ( + "github.com/ryogrid/SamehadaDB/lib/samehada/samehada_util" + "github.com/ryogrid/SamehadaDB/lib/storage/buffer" + "github.com/ryogrid/SamehadaDB/lib/storage/index/index_common" + "github.com/ryogrid/SamehadaDB/lib/storage/page" + "github.com/ryogrid/SamehadaDB/lib/storage/page/skip_list_page" + "github.com/ryogrid/SamehadaDB/lib/types" + blink_tree "github.com/ryogrid/bltree-go-for-embedding" +) + +type BTreeIterator struct { + bltr *blink_tree.BLTree + bpm *buffer.BufferPoolManager + curNode *skip_list_page.SkipListBlockPage + curEntry *index_common.IndexEntry + rangeStartKey *types.Value + rangeEndKey *types.Value + keyType types.TypeID + entryList []*index_common.IndexEntry + curEntryIdx int32 +} + +func NewSkipListIterator(bltr *blink_tree.BLTree, rangeStartKey *types.Value, rangeEndKey *types.Value) *BTreeIterator { + ret := new(BTreeIterator) + + // TODO: (SDB) need to implement this + panic("Not implemented yet") + //headerPage := sl.getHeaderPage() + // + //ret.sl = sl + //ret.bpm = sl.bpm + // + //ret.rangeStartKey = rangeStartKey + //ret.rangeEndKey = rangeEndKey + //ret.keyType = headerPage.GetKeyType() + //ret.entryList = make([]*skip_list_page.IndexEntry, 0) + // + //ret.initRIDList(sl) + + return ret +} + +func (itr *BTreeIterator) initRIDList(bltr *blink_tree.BLTree) { + // TODO: (SDB) need to implement this + panic("Not implemented yet") +} + +func (itr *BTreeIterator) Next() (done bool, err error, key *types.Value, rid *page.RID) { + // TODO: (SDB) need to implement this + panic("Not implemented yet") + if itr.curEntryIdx < int32(len(itr.entryList)) { + ret := itr.entryList[itr.curEntryIdx] + itr.curEntryIdx++ + tmpRID := samehada_util.UnpackUint64toRID(ret.Value) + return false, nil, samehada_util.GetPonterOfValue(ret.Key), &tmpRID + } else { + return true, nil, nil, nil + } +} diff --git a/lib/container/btree/btree_test.go b/lib/container/btree/btree_test.go index 273e1568..ea805d63 100644 --- a/lib/container/btree/btree_test.go +++ b/lib/container/btree/btree_test.go @@ -112,7 +112,7 @@ func TestBLTree_deleteMany_embedding(t *testing.T) { } for i := range keys { - if err := bltree.InsertKey(keys[i], 0, [blink_tree.BtId]byte{0, 0, 0, 0, 0, 0}, true); err != blink_tree.BLTErrOk { + if err := bltree.InsertKey(keys[i], 0, [blink_tree.BtId]byte{0, 0, 0, 0, 0, 0, 0, 0}, true); err != blink_tree.BLTErrOk { t.Errorf("InsertKey() = %v, want %v", err, blink_tree.BLTErrOk) } if i%2 == 0 { @@ -128,8 +128,8 @@ func TestBLTree_deleteMany_embedding(t *testing.T) { t.Errorf("FindKey() = %v, want %v, key %v", found, -1, keys[i]) } } else { - if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 { - t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i]) + if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 { + t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i]) } } } @@ -154,7 +154,7 @@ func TestBLTree_deleteAll_embedding(t *testing.T) { } for i := range keys { - if err := bltree.InsertKey(keys[i], 0, [blink_tree.BtId]byte{0, 0, 0, 0, 0, 0}, true); err != blink_tree.BLTErrOk { + if err := bltree.InsertKey(keys[i], 0, [blink_tree.BtId]byte{0, 0, 0, 0, 0, 0, 0, 0}, true); err != blink_tree.BLTErrOk { t.Errorf("InsertKey() = %v, want %v", err, blink_tree.BLTErrOk) } } @@ -216,8 +216,8 @@ func TestBLTree_deleteManyConcurrently_embedding(t *testing.T) { panic("FindKey() != -1") } } else { - if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 { - t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i]) + if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 { + t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i]) panic("FindKey() != 6") } } @@ -245,8 +245,8 @@ func TestBLTree_deleteManyConcurrently_embedding(t *testing.T) { t.Errorf("FindKey() = %v, want %v, key %v", found, -1, keys[i]) } } else { - if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 { - t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i]) + if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 { + t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i]) } } } @@ -330,8 +330,8 @@ func TestBLTree_deleteInsertRangeScanConcurrently_embedding(t *testing.T) { } rangeScanCheck(keys[i]) } else { - if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 { - t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i]) + if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 { + t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i]) panic("FindKey() != 6") } rangeScanCheck(keys[i]) @@ -363,8 +363,8 @@ func TestBLTree_deleteInsertRangeScanConcurrently_embedding(t *testing.T) { } } } else { - if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 { - t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i]) + if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 { + t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i]) } } } @@ -428,8 +428,8 @@ func TestBLTree_deleteManyConcurrentlyShuffle_embedding(t *testing.T) { panic("FindKey() != -1") } } else { - if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 { - t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i]) + if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 { + t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i]) panic("FindKey() != 6") } } @@ -457,8 +457,8 @@ func TestBLTree_deleteManyConcurrentlyShuffle_embedding(t *testing.T) { t.Errorf("FindKey() = %v, want %v, key %v", found, -1, keys[i]) } } else { - if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 { - t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i]) + if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 { + t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i]) } } } diff --git a/lib/container/skip_list/skip_list_iterator.go b/lib/container/skip_list/skip_list_iterator.go index d78588f4..e481ffb3 100644 --- a/lib/container/skip_list/skip_list_iterator.go +++ b/lib/container/skip_list/skip_list_iterator.go @@ -3,6 +3,7 @@ package skip_list import ( "github.com/ryogrid/SamehadaDB/lib/samehada/samehada_util" "github.com/ryogrid/SamehadaDB/lib/storage/buffer" + "github.com/ryogrid/SamehadaDB/lib/storage/index/index_common" "github.com/ryogrid/SamehadaDB/lib/storage/page" "github.com/ryogrid/SamehadaDB/lib/storage/page/skip_list_page" "github.com/ryogrid/SamehadaDB/lib/types" @@ -12,11 +13,11 @@ type SkipListIterator struct { sl *SkipList bpm *buffer.BufferPoolManager curNode *skip_list_page.SkipListBlockPage - curEntry *skip_list_page.SkipListPair + curEntry *index_common.IndexEntry rangeStartKey *types.Value rangeEndKey *types.Value keyType types.TypeID - entryList []*skip_list_page.SkipListPair + entryList []*index_common.IndexEntry curEntryIdx int32 } @@ -31,7 +32,7 @@ func NewSkipListIterator(sl *SkipList, rangeStartKey *types.Value, rangeEndKey * ret.rangeStartKey = rangeStartKey ret.rangeEndKey = rangeEndKey ret.keyType = headerPage.GetKeyType() - ret.entryList = make([]*skip_list_page.SkipListPair, 0) + ret.entryList = make([]*index_common.IndexEntry, 0) ret.initRIDList(sl) diff --git a/lib/container/skip_list/skip_list_test/skip_list_test.go b/lib/container/skip_list/skip_list_test/skip_list_test.go index 87b482f4..cfc14efe 100644 --- a/lib/container/skip_list/skip_list_test/skip_list_test.go +++ b/lib/container/skip_list/skip_list_test/skip_list_test.go @@ -6,6 +6,7 @@ import ( "github.com/ryogrid/SamehadaDB/lib/container/skip_list" "github.com/ryogrid/SamehadaDB/lib/samehada" "github.com/ryogrid/SamehadaDB/lib/samehada/samehada_util" + "github.com/ryogrid/SamehadaDB/lib/storage/index/index_common" "github.com/ryogrid/SamehadaDB/lib/storage/page/skip_list_page" testingpkg "github.com/ryogrid/SamehadaDB/lib/testing/testing_assert" "github.com/ryogrid/SamehadaDB/lib/types" @@ -26,7 +27,7 @@ func TestSerializationOfSkipLisBlockPage(t *testing.T) { shi := samehada.NewSamehadaInstance(t.Name(), common.BufferPoolMaxFrameNumForTest) bpm := shi.GetBufferPoolManager() - bpage := skip_list_page.NewSkipListBlockPage(bpm, 1, skip_list_page.SkipListPair{ + bpage := skip_list_page.NewSkipListBlockPage(bpm, 1, index_common.IndexEntry{ Key: types.NewInteger(math.MinInt32), Value: 0, }) @@ -39,7 +40,7 @@ func TestSerializationOfSkipLisBlockPage(t *testing.T) { bpage.SetFreeSpacePointer(common.PageSize - 9) // EntryCnt is incremented to 2 // freeSpacePointer is decremented size of entry (1+2+7+8 => 18) - bpage.SetEntry(1, &skip_list_page.SkipListPair{types.NewVarchar("abcdeff"), 12345}) + bpage.SetEntry(1, &index_common.IndexEntry{types.NewVarchar("abcdeff"), 12345}) testingpkg.SimpleAssert(t, bpage.GetPageId() == 7) testingpkg.SimpleAssert(t, bpage.GetLSN() == 9) @@ -90,26 +91,26 @@ func TestInnerInsertDeleteOfBlockPageSimple(t *testing.T) { // ---------- test RemoveInner -------- // setup a page - bpage1 := skip_list_page.NewSkipListBlockPage(bpm, 1, skip_list_page.SkipListPair{ + bpage1 := skip_list_page.NewSkipListBlockPage(bpm, 1, index_common.IndexEntry{ Key: types.NewVarchar("abcd"), Value: 1, }) - initialEntries := make([]*skip_list_page.SkipListPair, 0) + initialEntries := make([]*index_common.IndexEntry, 0) initialEntries = append(initialEntries, bpage1.GetEntry(0, types.Varchar)) - initialEntries = append(initialEntries, &skip_list_page.SkipListPair{ + initialEntries = append(initialEntries, &index_common.IndexEntry{ Key: types.NewVarchar("abcde"), Value: 2, }) - initialEntries = append(initialEntries, &skip_list_page.SkipListPair{ + initialEntries = append(initialEntries, &index_common.IndexEntry{ Key: types.NewVarchar("abcdef"), Value: 3, }) - initialEntries = append(initialEntries, &skip_list_page.SkipListPair{ + initialEntries = append(initialEntries, &index_common.IndexEntry{ Key: types.NewVarchar("abcdefg"), Value: 4, }) - initialEntries = append(initialEntries, &skip_list_page.SkipListPair{ + initialEntries = append(initialEntries, &index_common.IndexEntry{ Key: types.NewVarchar("abcdefgh"), Value: 5, }) @@ -129,33 +130,33 @@ func TestInnerInsertDeleteOfBlockPageSimple(t *testing.T) { // ---------- test InsertInner -------- // setup a page - bpage2 := skip_list_page.NewSkipListBlockPage(bpm, 1, skip_list_page.SkipListPair{ + bpage2 := skip_list_page.NewSkipListBlockPage(bpm, 1, index_common.IndexEntry{ Key: types.NewVarchar("abcd"), Value: 0, }) - initialEntries = make([]*skip_list_page.SkipListPair, 0) + initialEntries = make([]*index_common.IndexEntry, 0) initialEntries = append(initialEntries, bpage2.GetEntry(0, types.Varchar)) - initialEntries = append(initialEntries, &skip_list_page.SkipListPair{ + initialEntries = append(initialEntries, &index_common.IndexEntry{ Key: types.NewVarchar("abcde"), Value: 1, }) - initialEntries = append(initialEntries, &skip_list_page.SkipListPair{ + initialEntries = append(initialEntries, &index_common.IndexEntry{ Key: types.NewVarchar("abcdef"), Value: 2, }) bpage2.SetEntries(initialEntries) // insert entries - bpage2.InsertInner(-1, &skip_list_page.SkipListPair{ + bpage2.InsertInner(-1, &index_common.IndexEntry{ Key: types.NewVarchar("abc"), Value: 0, }) - bpage2.InsertInner(2, &skip_list_page.SkipListPair{ + bpage2.InsertInner(2, &index_common.IndexEntry{ Key: types.NewVarchar("abcdee"), Value: 22, }) - bpage2.InsertInner(4, &skip_list_page.SkipListPair{ + bpage2.InsertInner(4, &index_common.IndexEntry{ Key: types.NewVarchar("abcdeff"), Value: 33, }) @@ -190,20 +191,20 @@ func TestBSearchOfSkipLisBlockPage(t *testing.T) { shi := samehada.NewSamehadaInstance(t.Name(), common.BufferPoolMaxFrameNumForTest) bpm := shi.GetBufferPoolManager() - bpage := skip_list_page.NewSkipListBlockPage(bpm, 1, skip_list_page.SkipListPair{ + bpage := skip_list_page.NewSkipListBlockPage(bpm, 1, index_common.IndexEntry{ Key: types.NewInteger(math.MinInt32), Value: 0, }) // ------- when element num is even number ----- - bpage.SetEntries(make([]*skip_list_page.SkipListPair, 0)) - bpage.SetEntries(append(bpage.GetEntries(types.Integer), &skip_list_page.SkipListPair{ + bpage.SetEntries(make([]*index_common.IndexEntry, 0)) + bpage.SetEntries(append(bpage.GetEntries(types.Integer), &index_common.IndexEntry{ Key: types.NewInteger(math.MinInt32), Value: 0, })) // set entries for ii := 1; ii < 50; ii++ { - bpage.SetEntries(append(bpage.GetEntries(types.Integer), &skip_list_page.SkipListPair{types.NewInteger(int32(ii * 10)), uint64(ii * 10)})) + bpage.SetEntries(append(bpage.GetEntries(types.Integer), &index_common.IndexEntry{types.NewInteger(int32(ii * 10)), uint64(ii * 10)})) } bpage.SetEntryCnt(int32(len(bpage.GetEntries(types.Integer)))) @@ -218,14 +219,14 @@ func TestBSearchOfSkipLisBlockPage(t *testing.T) { } // ------- when element num is odd number ----- - bpage.SetEntries(make([]*skip_list_page.SkipListPair, 0)) - bpage.SetEntries(append(bpage.GetEntries(types.Integer), &skip_list_page.SkipListPair{ + bpage.SetEntries(make([]*index_common.IndexEntry, 0)) + bpage.SetEntries(append(bpage.GetEntries(types.Integer), &index_common.IndexEntry{ Key: types.NewInteger(math.MinInt32), Value: 0, })) // set entries for ii := 1; ii < 51; ii++ { - bpage.SetEntries(append(bpage.GetEntries(types.Integer), &skip_list_page.SkipListPair{types.NewInteger(int32(ii * 10)), uint64(ii * 10)})) + bpage.SetEntries(append(bpage.GetEntries(types.Integer), &index_common.IndexEntry{types.NewInteger(int32(ii * 10)), uint64(ii * 10)})) } bpage.SetEntryCnt(int32(len(bpage.GetEntries(types.Integer)))) @@ -251,14 +252,14 @@ func TestBSearchOfSkipLisBlockPage2(t *testing.T) { shi := samehada.NewSamehadaInstance(t.Name(), common.BufferPoolMaxFrameNumForTest) bpm := shi.GetBufferPoolManager() - bpage := skip_list_page.NewSkipListBlockPage(bpm, 1, skip_list_page.SkipListPair{ + bpage := skip_list_page.NewSkipListBlockPage(bpm, 1, index_common.IndexEntry{ Key: types.NewInteger(math.MinInt32), Value: 0, }) // ------- when element num is even number ----- - bpage.SetEntries(make([]*skip_list_page.SkipListPair, 0)) - bpage.SetEntries(append(bpage.GetEntries(types.Integer), &skip_list_page.SkipListPair{ + bpage.SetEntries(make([]*index_common.IndexEntry, 0)) + bpage.SetEntries(append(bpage.GetEntries(types.Integer), &index_common.IndexEntry{ Key: types.NewInteger(math.MinInt32), Value: 0, })) @@ -284,8 +285,8 @@ func TestBSearchOfSkipLisBlockPage2(t *testing.T) { // ------- when element num is odd number ----- bpage.WLatch() - bpage.SetEntries(make([]*skip_list_page.SkipListPair, 0)) - bpage.SetEntries(append(bpage.GetEntries(types.Integer), &skip_list_page.SkipListPair{ + bpage.SetEntries(make([]*index_common.IndexEntry, 0)) + bpage.SetEntries(append(bpage.GetEntries(types.Integer), &index_common.IndexEntry{ Key: types.NewInteger(math.MinInt32), Value: 0, })) diff --git a/lib/execution/executors/executor_test/btree_index_executor_test.go b/lib/execution/executors/executor_test/btree_index_executor_test.go new file mode 100644 index 00000000..c54d4a7a --- /dev/null +++ b/lib/execution/executors/executor_test/btree_index_executor_test.go @@ -0,0 +1,122 @@ +package executor_test + +import ( + "fmt" + "github.com/ryogrid/SamehadaDB/lib/catalog" + "github.com/ryogrid/SamehadaDB/lib/common" + "github.com/ryogrid/SamehadaDB/lib/samehada" + "github.com/ryogrid/SamehadaDB/lib/storage/index/index_constants" + "github.com/ryogrid/SamehadaDB/lib/storage/table/column" + "github.com/ryogrid/SamehadaDB/lib/storage/table/schema" + testingpkg "github.com/ryogrid/SamehadaDB/lib/testing/testing_assert" + "github.com/ryogrid/SamehadaDB/lib/types" + "os" + "testing" +) + +func testKeyDuplicateInsertDeleteWithBTreeIndex[T float32 | int32 | string](t *testing.T, keyType types.TypeID) { + if !common.EnableOnMemStorage { + os.Remove(t.Name() + ".db") + os.Remove(t.Name() + ".log") + } + + shi := samehada.NewSamehadaInstance(t.Name(), 500) + shi.GetLogManager().ActivateLogging() + testingpkg.Assert(t, shi.GetLogManager().IsEnabledLogging(), "") + fmt.Println("System logging is active.") + txnMgr := shi.GetTransactionManager() + + txn := txnMgr.Begin(nil) + + c := catalog.BootstrapCatalog(shi.GetBufferPoolManager(), shi.GetLogManager(), shi.GetLockManager(), txn) + + columnA := column.NewColumn("account_id", keyType, true, index_constants.INDEX_KIND_BTREE, types.PageID(-1), nil) + columnB := column.NewColumn("balance", types.Integer, true, index_constants.INDEX_KIND_BTREE, types.PageID(-1), nil) + schema_ := schema.NewSchema([]*column.Column{columnA, columnB}) + tableMetadata := c.CreateTable("test_1", schema_, txn) + + txnMgr.Commit(c, txn) + + txn = txnMgr.Begin(nil) + + var accountId interface{} + switch keyType { + case types.Integer: + accountId = int32(10) + case types.Float: + accountId = float32(-5.2) + case types.Varchar: + accountId = "duplicateTest" + default: + panic("unsuppoted value type") + } + + insPlan1 := createSpecifiedValInsertPlanNode(accountId.(T), int32(100), c, tableMetadata, keyType) + result := executePlan(c, shi.GetBufferPoolManager(), txn, insPlan1) + insPlan2 := createSpecifiedValInsertPlanNode(accountId.(T), int32(101), c, tableMetadata, keyType) + result = executePlan(c, shi.GetBufferPoolManager(), txn, insPlan2) + insPlan3 := createSpecifiedValInsertPlanNode(accountId.(T), int32(102), c, tableMetadata, keyType) + result = executePlan(c, shi.GetBufferPoolManager(), txn, insPlan3) + + txnMgr.Commit(c, txn) + + txn = txnMgr.Begin(nil) + + //rangeScanP := createSpecifiedRangeScanPlanNode[T](c, tableMetadata, keyType, 0, nil, nil, index_constants.INDEX_KIND_BTREE) + //results := executePlan(c, shi.GetBufferPoolManager(), txn, rangeScanP) + //for _, foundVal := range results { + // fmt.Println(foundVal.GetValue(tableMetadata.Schema(), 0).ToString()) + //} + + scanP := createSpecifiedPointScanPlanNode(accountId.(T), c, tableMetadata, keyType, index_constants.INDEX_KIND_BTREE) + result = executePlan(c, shi.GetBufferPoolManager(), txn, scanP) + testingpkg.Assert(t, len(result) == 3, "duplicated key point scan got illegal results.") + rid1 := result[0].GetRID() + val0_1 := result[0].GetValue(tableMetadata.Schema(), 0) + val0_2 := result[0].GetValue(tableMetadata.Schema(), 1) + fmt.Println(val0_1, val0_2) + rid2 := result[1].GetRID() + rid3 := result[2].GetRID() + fmt.Printf("%v %v %v\n", *rid1, *rid2, *rid3) + + for _, foundTuple := range result { + val := foundTuple.GetValue(tableMetadata.Schema(), 0) + fmt.Println(val.ToString()) + } + + indexCol1 := tableMetadata.GetIndex(0) + indexCol2 := tableMetadata.GetIndex(1) + + indexCol1.DeleteEntry(result[0], *rid1, txn) + indexCol2.DeleteEntry(result[0], *rid1, txn) + scanP = createSpecifiedPointScanPlanNode(accountId.(T), c, tableMetadata, keyType, index_constants.INDEX_KIND_BTREE) + result = executePlan(c, shi.GetBufferPoolManager(), txn, scanP) + testingpkg.Assert(t, len(result) == 2, "duplicated key point scan got illegal results.") + + indexCol1.DeleteEntry(result[0], *rid2, txn) + indexCol2.DeleteEntry(result[0], *rid2, txn) + scanP = createSpecifiedPointScanPlanNode(accountId.(T), c, tableMetadata, keyType, index_constants.INDEX_KIND_BTREE) + result = executePlan(c, shi.GetBufferPoolManager(), txn, scanP) + testingpkg.Assert(t, len(result) == 1, "duplicated key point scan got illegal results.") + + indexCol1.DeleteEntry(result[0], *rid3, txn) + indexCol2.DeleteEntry(result[0], *rid3, txn) + scanP = createSpecifiedPointScanPlanNode(accountId.(T), c, tableMetadata, keyType, index_constants.INDEX_KIND_BTREE) + result = executePlan(c, shi.GetBufferPoolManager(), txn, scanP) + testingpkg.Assert(t, len(result) == 0, "duplicated key point scan got illegal results.") + + txnMgr.Commit(c, txn) + shi.Shutdown(samehada.ShutdownPatternCloseFiles) +} + +func TestKeyDuplicateInsertDeleteWithBTreeIndexInt(t *testing.T) { + testKeyDuplicateInsertDeleteWithBTreeIndex[int32](t, types.Integer) +} + +func TestKeyDuplicateInsertDeleteWithBTreeIndexFloat(t *testing.T) { + testKeyDuplicateInsertDeleteWithBTreeIndex[float32](t, types.Float) +} + +func TestKeyDuplicateInsertDeleteWithBTreeIndexVarchar(t *testing.T) { + testKeyDuplicateInsertDeleteWithBTreeIndex[string](t, types.Varchar) +} diff --git a/lib/execution/executors/executor_test/executor_test.go b/lib/execution/executors/executor_test/executor_test.go index 33734ac8..bb2da542 100644 --- a/lib/execution/executors/executor_test/executor_test.go +++ b/lib/execution/executors/executor_test/executor_test.go @@ -2221,7 +2221,7 @@ func TestInsertAndSpecifiedColumnUpdatePageMoveRecovery(t *testing.T) { log_mgr) txn = txn_mgr.Begin(nil) - c = catalog.RecoveryCatalogFromCatalogPage(bpm, log_mgr, lock_mgr, txn) + c = catalog.RecoveryCatalogFromCatalogPage(bpm, log_mgr, lock_mgr, txn, true) tableMetadata = c.GetTableByName("test_1") executorContext = executors.NewExecutorContext(c, bpm, txn) @@ -2396,7 +2396,7 @@ func TestInsertAndSpecifiedColumnUpdatePageMoveOccurOnRecovery(t *testing.T) { log_mgr) txn = txn_mgr.Begin(nil) - c = catalog.RecoveryCatalogFromCatalogPage(bpm, log_mgr, lock_mgr, txn) + c = catalog.RecoveryCatalogFromCatalogPage(bpm, log_mgr, lock_mgr, txn, true) tableMetadata = c.GetTableByName("test_1") executorContext = executors.NewExecutorContext(c, bpm, txn) @@ -2809,7 +2809,7 @@ func TestDeallocatedPageReuseAfterRelaunchGraceful(t *testing.T) { log_mgr) txn = txn_mgr.Begin(nil) - c = catalog.RecoveryCatalogFromCatalogPage(bpm, log_mgr, lock_mgr, txn) + c = catalog.RecoveryCatalogFromCatalogPage(bpm, log_mgr, lock_mgr, txn, true) tableMetadata = c.GetTableByName("test_1") executorContext = executors.NewExecutorContext(c, bpm, txn) @@ -2964,7 +2964,7 @@ func TestDeallocatedPageReuseAfterRelaunchByCrash(t *testing.T) { log_mgr) txn = txn_mgr.Begin(nil) - c = catalog.RecoveryCatalogFromCatalogPage(bpm, log_mgr, lock_mgr, txn) + c = catalog.RecoveryCatalogFromCatalogPage(bpm, log_mgr, lock_mgr, txn, false) tableMetadata = c.GetTableByName("test_1") executorContext = executors.NewExecutorContext(c, bpm, txn) diff --git a/lib/execution/executors/executor_test/skiplist_index_executor_test.go b/lib/execution/executors/executor_test/skiplist_index_executor_test.go index 7ba51730..c3caf674 100644 --- a/lib/execution/executors/executor_test/skiplist_index_executor_test.go +++ b/lib/execution/executors/executor_test/skiplist_index_executor_test.go @@ -117,7 +117,7 @@ func getNotDupWithAccountRandomPrimitivVal[T int32 | float32 | string](keyType t return retVal } -func testParallelTxnsQueryingSkipListIndexUsedColumns[T int32 | float32 | string](t *testing.T, keyType types.TypeID, stride int32, opTimes int32, seedVal int32, initialEntryNum int32, bpoolSize int32, indexKind index_constants.IndexKind, execType int32, threadNum int) { +func testParallelTxnsQueryingIndexUsedColumns[T int32 | float32 | string](t *testing.T, keyType types.TypeID, stride int32, opTimes int32, seedVal int32, initialEntryNum int32, bpoolSize int32, indexKind index_constants.IndexKind, execType int32, threadNum int) { common.ShPrintf(common.DEBUG_INFO, "start of testParallelTxnsQueryingUniqSkipListIndexUsedColumns stride=%d opTimes=%d seedVal=%d initialEntryNum=%d bpoolSize=%d ====================================================\n", stride, opTimes, seedVal, initialEntryNum, bpoolSize) @@ -1252,24 +1252,24 @@ func testSkipListParallelTxnStrideRoot[T int32 | float32 | string](t *testing.T, //testParallelTxnsQueryingUniqSkipListIndexUsedColumns[T](t, keyType, 400, 30000, 13, 0, bpoolSize, index_constants.INDEX_KIND_UNIQ_SKIP_LIST, PARALLEL_EXEC, 20) //testParallelTxnsQueryingUniqSkipListIndexUsedColumns[T](t, keyType, 400, 30000, 13, 0, bpoolSize, index_constants.INDEX_KIND_UNIQ_SKIP_LIST, PARALLEL_EXEC, 20) - //testParallelTxnsQueryingSkipListIndexUsedColumns[T](t, keyType, 400, 30000, 13, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, PARALLEL_EXEC, 20) - //testParallelTxnsQueryingSkipListIndexUsedColumns[T](t, keyType, 400, 30000, 13, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, SERIAL_EXEC, 20) - //testParallelTxnsQueryingSkipListIndexUsedColumns[T](t, keyType, 400, 300, 13, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, SERIAL_EXEC, 20) - //testParallelTxnsQueryingSkipListIndexUsedColumns[T](t, keyType, 400, 3000, 13, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, SERIAL_EXEC, 20) - testParallelTxnsQueryingSkipListIndexUsedColumns[T](t, keyType, 400, 3000, 13, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, PARALLEL_EXEC, 20) + //testParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 30000, 13, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, PARALLEL_EXEC, 20) + //testParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 30000, 13, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, SERIAL_EXEC, 20) + //testParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 300, 13, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, SERIAL_EXEC, 20) + //testParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 3000, 13, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, SERIAL_EXEC, 20) + testParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 3000, 13, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, PARALLEL_EXEC, 20) - //testParallelTxnsQueryingSkipListIndexUsedColumns[T](t, keyType, 400, 3000, 13, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, PARALLEL_EXEC, 20) + //testParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 3000, 13, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, PARALLEL_EXEC, 20) case types.Float: //testParallelTxnsQueryingUniqSkipListIndexUsedColumns[T](t, keyType, 400, 30000, 13, 0, bpoolSize, index_constants.INDEX_KIND_UNIQ_SKIP_LIST, PARALLEL_EXEC, 20) - testParallelTxnsQueryingSkipListIndexUsedColumns[T](t, keyType, 240, 1000, 13, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, PARALLEL_EXEC, 20) + testParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 240, 1000, 13, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, PARALLEL_EXEC, 20) case types.Varchar: //testParallelTxnsQueryingUniqSkipListIndexUsedColumns[T](t, keyType, 400, 400, 13, 0, bpoolSize, index_constants.INDEX_KIND_INVALID, PARALLEL_EXEC, 20) //testParallelTxnsQueryingUniqSkipListIndexUsedColumns[T](t, keyType, 400, 3000, 13, 0, bpoolSize, index_constants.INDEX_KIND_UNIQ_SKIP_LIST, PARALLEL_EXEC, 20) //testParallelTxnsQueryingUniqSkipListIndexUsedColumns[T](t, keyType, 400, 90000, 17, 0, bpoolSize, index_constants.INDEX_KIND_UNIQ_SKIP_LIST, PARALLEL_EXEC, 20) - //testParallelTxnsQueryingSkipListIndexUsedColumns[T](t, keyType, 400, 5000, 17, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, PARALLEL_EXEC, 20) - testParallelTxnsQueryingSkipListIndexUsedColumns[T](t, keyType, 400, 5000, 17, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, PARALLEL_EXEC, 20) + //testParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 5000, 17, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, PARALLEL_EXEC, 20) + testParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 5000, 17, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, PARALLEL_EXEC, 20) //testParallelTxnsQueryingUniqSkipListIndexUsedColumns[T](t, keyType, 400, 50000, 17, 0, bpoolSize, index_constants.INDEX_KIND_UNIQ_SKIP_LIST, PARALLEL_EXEC, 20) //testParallelTxnsQueryingUniqSkipListIndexUsedColumns[T](t, keyType, 400, 200000, 11, 0, bpoolSize, index_constants.INDEX_KIND_UNIQ_SKIP_LIST, PARALLEL_EXEC, 20) diff --git a/lib/execution/executors/executor_test/uniq_skiplist_index_executor_test.go b/lib/execution/executors/executor_test/uniq_skiplist_index_executor_test.go index 40c626be..7d27b43a 100644 --- a/lib/execution/executors/executor_test/uniq_skiplist_index_executor_test.go +++ b/lib/execution/executors/executor_test/uniq_skiplist_index_executor_test.go @@ -684,7 +684,7 @@ func createSpecifiedPointScanPlanNode[T int32 | float32 | string](getKeyVal T, c switch indexKind { case index_constants.INDEX_KIND_INVALID: skipListPointScanP = plans.NewSeqScanPlanNode(c, tm.Schema(), expression_.(*expression.Comparison), tm.OID()) - case index_constants.INDEX_KIND_UNIQ_SKIP_LIST, index_constants.INDEX_KIND_SKIP_LIST: + case index_constants.INDEX_KIND_UNIQ_SKIP_LIST, index_constants.INDEX_KIND_SKIP_LIST, index_constants.INDEX_KIND_BTREE: skipListPointScanP = plans.NewPointScanWithIndexPlanNode(c, tm.Schema(), expression_.(*expression.Comparison), tm.OID()) default: panic("not implemented!") @@ -700,7 +700,7 @@ func createSpecifiedRangeScanPlanNode[T int32 | float32 | string](c *catalog.Cat switch indexKind { case index_constants.INDEX_KIND_INVALID: skipListRangeScanP = plans.NewSeqScanPlanNode(c, tm.Schema(), nil, tm.OID()) - case index_constants.INDEX_KIND_UNIQ_SKIP_LIST, index_constants.INDEX_KIND_SKIP_LIST: + case index_constants.INDEX_KIND_UNIQ_SKIP_LIST, index_constants.INDEX_KIND_SKIP_LIST, index_constants.INDEX_KIND_BTREE: if rangeStartKey != nil { startVal = samehada_util.GetPonterOfValue(types.NewValue(*rangeStartKey)) } diff --git a/lib/execution/executors/range_scan_with_index_executor.go b/lib/execution/executors/range_scan_with_index_executor.go index 09730fb5..b266e984 100644 --- a/lib/execution/executors/range_scan_with_index_executor.go +++ b/lib/execution/executors/range_scan_with_index_executor.go @@ -82,11 +82,23 @@ func (e *RangeScanWithIndexExecutor) Next() (*tuple.Tuple, Done, error) { } case *index.SkipListIndex: orgKey := samehada_util.ExtractOrgKeyFromDicOrderComparableEncodedVarchar(key, orgKeyType) + if !curKeyVal.CompareEquals(*orgKey) { // column value corresponding index key is updated e.txn.SetState(access.ABORTED) return nil, true, errors.New("detect value update after iterator created. changes transaction state to aborted.") } + case *index.BTreeIndex: + //orgKey := samehada_util.ExtractOrgKeyFromDicOrderComparableEncodedVarchar(key, orgKeyType) + + //if !curKeyVal.CompareEquals(*orgKey) { + + // when BTreeIndex is used, key is original key + if !curKeyVal.CompareEquals(*key) { + // column value corresponding index key is updated + e.txn.SetState(access.ABORTED) + return nil, true, errors.New("detect value update after iterator created. changes transaction state to aborted.") + } } // check predicate diff --git a/lib/go.mod b/lib/go.mod index c4771b60..98a696d7 100644 --- a/lib/go.mod +++ b/lib/go.mod @@ -12,7 +12,7 @@ require ( github.com/notEpsilon/go-pair v0.0.0-20221220200415-e91ef28c6c0b github.com/pingcap/parser v0.0.0-20200623164729-3a18f1e5dceb github.com/pingcap/tidb v1.1.0-beta.0.20200630082100-328b6d0a955c - github.com/ryogrid/bltree-go-for-embedding v1.0.2 + github.com/ryogrid/bltree-go-for-embedding v1.0.4 github.com/spaolacci/murmur3 v1.1.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 ) diff --git a/lib/go.sum b/lib/go.sum index d828bd85..9530ce8e 100644 --- a/lib/go.sum +++ b/lib/go.sum @@ -448,8 +448,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryogrid/bltree-go-for-embedding v1.0.2 h1:GquFei9pmPy4GK9YBGwXITcJFCnRjWTX+BDpaIVxJLI= -github.com/ryogrid/bltree-go-for-embedding v1.0.2/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE= +github.com/ryogrid/bltree-go-for-embedding v1.0.4 h1:LAvotdhSovWV3T2m8X9LtcKt7E+NrndEcLgUCunHwLg= +github.com/ryogrid/bltree-go-for-embedding v1.0.4/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= diff --git a/lib/samehada/samehada.go b/lib/samehada/samehada.go index cfa0bbfd..f80fbd8d 100644 --- a/lib/samehada/samehada.go +++ b/lib/samehada/samehada.go @@ -75,6 +75,10 @@ func reconstructIndexDataOfATbl(t *catalog.TableMetadata, c *catalog.Catalog, dm // do nothing here // (Since SkipList index can't reuse past allocated pages, data clear of allocated pages // are not needed...) + case index_constants.INDEX_KIND_BTREE: + // do nothing here + // (Since BTree index can't reuse past allocated pages, data clear of allocated pages + // are not needed...) default: panic("invalid index kind!") } @@ -147,7 +151,7 @@ func NewSamehadaDB(dbName string, memKBytes int) *SamehadaDB { } shi.log_manager.Flush() - c = catalog.RecoveryCatalogFromCatalogPage(shi.GetBufferPoolManager(), shi.GetLogManager(), shi.GetLockManager(), txn) + c = catalog.RecoveryCatalogFromCatalogPage(shi.GetBufferPoolManager(), shi.GetLogManager(), shi.GetLockManager(), txn, isGracefulShutdown) // if last shutdown is not gracefully done, all index data should be reconstructed if !isGracefulShutdown { diff --git a/lib/samehada/samehada_instance.go b/lib/samehada/samehada_instance.go index 72d1d77e..2baca55f 100644 --- a/lib/samehada/samehada_instance.go +++ b/lib/samehada/samehada_instance.go @@ -87,6 +87,7 @@ func (si *SamehadaInstance) Shutdown(shutdownPat ShutdownPattern) { si.disk_manager.RemoveLogFile() case ShutdownPatternCloseFiles: si.log_manager.Flush() + // TODO: (SDB) need to finalize BTreeIndex objects si.bpm.FlushAllDirtyPages() logRecord := recovery.NewLogRecordGracefulShutdown() si.log_manager.AppendLogRecord(logRecord) diff --git a/lib/samehada/samehada_util/samehada_util.go b/lib/samehada/samehada_util/samehada_util.go index 3525f192..ee948a3f 100644 --- a/lib/samehada/samehada_util/samehada_util.go +++ b/lib/samehada/samehada_util/samehada_util.go @@ -241,7 +241,7 @@ func encodeToDicOrderComparableBytes(orgVal interface{}, valType types.TypeID) [ buf := new(bytes.Buffer) binary.Write(buf, binary.BigEndian, u) return buf.Bytes() - case valType: + case types.Integer: i := orgVal.(int32) u := uint32(i) buf := new(bytes.Buffer) @@ -339,6 +339,22 @@ func ExtractOrgKeyFromDicOrderComparableEncodedVarchar(encodedVal *types.Value, } } +func ExtractOrgKeyFromDicOrderComparableEncodedBytes(buf []byte, valType types.TypeID) *types.Value { + switch valType { + case types.Integer: + retVal := types.NewValue(decodeFromDicOrderComparableBytes(buf[3:len(buf)-8], valType).(int32)) + return &retVal + case types.Float: + retVal := types.NewValue(decodeFromDicOrderComparableBytes(buf[3:len(buf)-8], valType).(float32)) + return &retVal + case types.Varchar: + orgStr := string(buf[:len(buf)-(4+8)]) + return GetPonterOfValue(types.NewVarchar(orgStr)) + default: + panic("not supported type") + } +} + func SHAssert(cond bool, msg string) { if !cond { panic(msg) diff --git a/lib/storage/index/btree_index.go b/lib/storage/index/btree_index.go new file mode 100644 index 00000000..2b18aad9 --- /dev/null +++ b/lib/storage/index/btree_index.go @@ -0,0 +1,197 @@ +package index + +import ( + "encoding/binary" + "github.com/ryogrid/SamehadaDB/lib/common" + "github.com/ryogrid/SamehadaDB/lib/container/btree" + "github.com/ryogrid/SamehadaDB/lib/recovery" + "github.com/ryogrid/SamehadaDB/lib/samehada/samehada_util" + "github.com/ryogrid/SamehadaDB/lib/storage/buffer" + "github.com/ryogrid/SamehadaDB/lib/storage/page" + "github.com/ryogrid/SamehadaDB/lib/storage/table/schema" + "github.com/ryogrid/SamehadaDB/lib/storage/tuple" + "github.com/ryogrid/SamehadaDB/lib/types" + blink_tree "github.com/ryogrid/bltree-go-for-embedding" + "math" + "sync" +) + +type BtreeIndexIterator struct { + itr *blink_tree.BLTreeItr + valType types.TypeID +} + +func NewBtreeIndexIterator(itr *blink_tree.BLTreeItr, valType types.TypeID) *BtreeIndexIterator { + return &BtreeIndexIterator{itr, valType} +} + +func (btreeItr *BtreeIndexIterator) Next() (done bool, err error, key *types.Value, rid *page.RID) { + ok, keyBytes, packedRID := btreeItr.itr.Next() + if ok == false { + return true, nil, nil, &page.RID{-1, 0} + } + uintRID := binary.BigEndian.Uint64(packedRID) + unpackedRID := samehada_util.UnpackUint64toRID(uintRID) + + // attach isNull flag and length of value due to these info is not stored in BLTree + keyLen := uint16(len(keyBytes) - 8) // 8 is length of packedRID + keyLenBuf := make([]byte, 2) + binary.LittleEndian.PutUint16(keyLenBuf, keyLen) + newKeyBytes := make([]byte, 0, len(keyBytes)+3) + newKeyBytes = append(newKeyBytes, 0) + newKeyBytes = append(newKeyBytes, keyLenBuf...) + newKeyBytes = append(newKeyBytes, keyBytes...) + + decodedKey := samehada_util.ExtractOrgKeyFromDicOrderComparableEncodedBytes(newKeyBytes, btreeItr.valType) + return false, nil, decodedKey, &unpackedRID +} + +type BTreeIndex struct { + container *blink_tree.BLTree + metadata *IndexMetadata + // idx of target column on table + col_idx uint32 + log_manager *recovery.LogManager + // UpdateEntry only get Write lock + updateMtx sync.RWMutex + // for call of Close method .... + bufMgr *blink_tree.BufMgr +} + +func NewBTreeIndex(metadata *IndexMetadata, buffer_pool_manager *buffer.BufferPoolManager, col_idx uint32, log_manager *recovery.LogManager, lastPageZeroId *int32) *BTreeIndex { + ret := new(BTreeIndex) + ret.metadata = metadata + + // BTreeIndex uses special technique to support key duplication with SkipList supporting unique key only + // for the thechnique, key type is fixed to Varchar (comparison is done on dict order as byte array) + + bufMgr := blink_tree.NewBufMgr(12, blink_tree.HASH_TABLE_ENTRY_CHAIN_LEN*common.MaxTxnThreadNum*2, btree.NewParentBufMgrImpl(buffer_pool_manager), lastPageZeroId) + ret.container = blink_tree.NewBLTree(bufMgr) + ret.col_idx = col_idx + ret.updateMtx = sync.RWMutex{} + ret.log_manager = log_manager + ret.bufMgr = bufMgr + return ret +} + +func (btidx *BTreeIndex) insertEntryInner(key *tuple.Tuple, rid page.RID, txn interface{}, isNoLock bool) { + tupleSchema_ := btidx.GetTupleSchema() + orgKeyVal := key.GetValue(tupleSchema_, btidx.col_idx) + + convedKeyVal := samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgKeyVal, &rid) + + if isNoLock == false { + btidx.updateMtx.RLock() + defer btidx.updateMtx.RUnlock() + } + + packedRID := samehada_util.PackRIDtoUint64(&rid) + var valBuf [8]byte + binary.BigEndian.PutUint64(valBuf[:], packedRID) + btidx.container.InsertKey(convedKeyVal.SerializeOnlyVal(), 0, valBuf, true) +} + +func (btidx *BTreeIndex) InsertEntry(key *tuple.Tuple, rid page.RID, txn interface{}) { + btidx.insertEntryInner(key, rid, txn, false) +} + +func (btidx *BTreeIndex) deleteEntryInner(key *tuple.Tuple, rid page.RID, txn interface{}, isNoLock bool) { + tupleSchema_ := btidx.GetTupleSchema() + orgKeyVal := key.GetValue(tupleSchema_, btidx.col_idx) + + convedKeyVal := samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgKeyVal, &rid) + + if isNoLock == false { + btidx.updateMtx.RLock() + defer btidx.updateMtx.RUnlock() + } + btidx.container.DeleteKey(convedKeyVal.SerializeOnlyVal(), 0) +} + +func (btidx *BTreeIndex) DeleteEntry(key *tuple.Tuple, rid page.RID, txn interface{}) { + btidx.deleteEntryInner(key, rid, txn, false) +} + +func (btidx *BTreeIndex) ScanKey(key *tuple.Tuple, txn interface{}) []page.RID { + tupleSchema_ := btidx.GetTupleSchema() + orgKeyVal := key.GetValue(tupleSchema_, btidx.col_idx) + smallestKeyVal := samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgKeyVal, &page.RID{0, 0}) + biggestKeyVal := samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgKeyVal, &page.RID{math.MaxInt32, math.MaxUint32}) + + btidx.updateMtx.RLock() + // Attention: returned itr's containing keys are string type Value which is constructed with byte arr of concatenated original key and value + rangeItr := btidx.container.GetRangeItr(smallestKeyVal.SerializeOnlyVal(), biggestKeyVal.SerializeOnlyVal()) + + retArr := make([]page.RID, 0) + for ok, _, packedRID := rangeItr.Next(); ok; ok, _, packedRID = rangeItr.Next() { + uintRID := binary.BigEndian.Uint64(packedRID) + retArr = append(retArr, samehada_util.UnpackUint64toRID(uintRID)) + } + btidx.updateMtx.RUnlock() + + return retArr +} + +func (btidx *BTreeIndex) UpdateEntry(oldKey *tuple.Tuple, oldRID page.RID, newKey *tuple.Tuple, newRID page.RID, txn interface{}) { + btidx.updateMtx.Lock() + defer btidx.updateMtx.Unlock() + btidx.deleteEntryInner(oldKey, oldRID, txn, true) + btidx.insertEntryInner(newKey, newRID, txn, true) +} + +// get iterator which iterates entry in key sorted order +// and iterates specified key range. +// when start_key arg is nil , start point is head of entry list. when end_key, end point is tail of the list +// Attention: returned itr's containing keys are string type Value which is constructed with byte arr of concatenated original key and value +func (btidx *BTreeIndex) GetRangeScanIterator(start_key *tuple.Tuple, end_key *tuple.Tuple, transaction interface{}) IndexRangeScanIterator { + tupleSchema_ := btidx.GetTupleSchema() + var smallestKeyVal *types.Value = nil + if start_key != nil { + orgStartKeyVal := start_key.GetValue(tupleSchema_, btidx.col_idx) + smallestKeyVal = samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgStartKeyVal, &page.RID{0, 0}) + } + + var biggestKeyVal *types.Value = nil + if end_key != nil { + orgEndKeyVal := end_key.GetValue(tupleSchema_, btidx.col_idx) + biggestKeyVal = samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgEndKeyVal, &page.RID{math.MaxInt32, math.MaxUint32}) + } + + btidx.updateMtx.RLock() + defer btidx.updateMtx.RUnlock() + var smalledKeyBytes []byte + var biggestKeyBytes []byte + + if smallestKeyVal != nil { + smalledKeyBytes = smallestKeyVal.SerializeOnlyVal() + } + if biggestKeyVal != nil { + biggestKeyBytes = biggestKeyVal.SerializeOnlyVal() + } + return NewBtreeIndexIterator(btidx.container.GetRangeItr(smalledKeyBytes, biggestKeyBytes), btidx.metadata.tuple_schema.GetColumn(btidx.col_idx).GetType()) +} + +// Return the metadata object associated with the index +func (btidx *BTreeIndex) GetMetadata() *IndexMetadata { return btidx.metadata } + +func (btidx *BTreeIndex) GetIndexColumnCount() uint32 { + return btidx.metadata.GetIndexColumnCount() +} + +func (btidx *BTreeIndex) GetName() *string { return btidx.metadata.GetName() } + +func (btidx *BTreeIndex) GetTupleSchema() *schema.Schema { + return btidx.metadata.GetTupleSchema() +} + +func (btidx *BTreeIndex) GetKeyAttrs() []uint32 { return btidx.metadata.GetKeyAttrs() } + +func (slidx *BTreeIndex) GetHeaderPageId() types.PageID { + return types.PageID(slidx.bufMgr.GetMappedPPageIdOfPageZero()) +} + +// call this at shutdown of the system +// to write out the state and allocated pages of the BLTree container to BPM +func (btidx *BTreeIndex) WriteOutContainerStateToBPM() { + btidx.bufMgr.Close() +} diff --git a/lib/storage/index/index_common/index_common.go b/lib/storage/index/index_common/index_common.go new file mode 100644 index 00000000..c0e44513 --- /dev/null +++ b/lib/storage/index/index_common/index_common.go @@ -0,0 +1,40 @@ +package index_common + +import ( + "bytes" + "encoding/binary" + "github.com/ryogrid/SamehadaDB/lib/types" +) + +const sizeEntryValue = uint32(8) + +type IndexEntry struct { + Key types.Value + Value uint64 +} + +func (ie IndexEntry) Serialize() []byte { + keyInBytes := ie.Key.Serialize() + valBuf := new(bytes.Buffer) + binary.Write(valBuf, binary.LittleEndian, ie.Value) + valInBytes := valBuf.Bytes() + + retBuf := new(bytes.Buffer) + retBuf.Write(keyInBytes) + retBuf.Write(valInBytes) + return retBuf.Bytes() +} + +func (ie IndexEntry) GetDataSize() uint32 { + keyInBytes := ie.Key.Serialize() + + return uint32(len(keyInBytes)) + sizeEntryValue +} + +func NewIndexEntryFromBytes(buf []byte, keyType types.TypeID) *IndexEntry { + dataLen := len(buf) + valPartOffset := dataLen - int(sizeEntryValue) + key := types.NewValueFromBytes(buf[:valPartOffset], keyType) + value := uint64(types.NewUInt64FromBytes(buf[valPartOffset:])) + return &IndexEntry{*key, value} +} diff --git a/lib/storage/index/index_constants/index_constants.go b/lib/storage/index/index_constants/index_constants.go index 8ac952be..de9597e3 100644 --- a/lib/storage/index/index_constants/index_constants.go +++ b/lib/storage/index/index_constants/index_constants.go @@ -7,4 +7,5 @@ const ( INDEX_KIND_UNIQ_SKIP_LIST INDEX_KIND_SKIP_LIST INDEX_KIND_HASH + INDEX_KIND_BTREE // B-link tree ) diff --git a/lib/storage/index/index_test/hash_table_index_test.go b/lib/storage/index/index_test/hash_table_index_test.go index b871276b..dc9b2316 100644 --- a/lib/storage/index/index_test/hash_table_index_test.go +++ b/lib/storage/index/index_test/hash_table_index_test.go @@ -174,7 +174,7 @@ func TestRecounstructionOfHashIndex(t *testing.T) { dman := shi.GetDiskManager() dman.GCLogFile() shi.GetLogManager().SetNextLSN(greatestLSN + 1) - c = catalog.RecoveryCatalogFromCatalogPage(shi.GetBufferPoolManager(), shi.GetLogManager(), shi.GetLockManager(), txn) + c = catalog.RecoveryCatalogFromCatalogPage(shi.GetBufferPoolManager(), shi.GetLogManager(), shi.GetLockManager(), txn, true) // reconstruct all index data of all column tableMetadata = c.GetTableByName("test_1") diff --git a/lib/storage/page/skip_list_page/skip_list_block_page.go b/lib/storage/page/skip_list_page/skip_list_block_page.go index 37f340bf..94ccc502 100644 --- a/lib/storage/page/skip_list_page/skip_list_block_page.go +++ b/lib/storage/page/skip_list_page/skip_list_block_page.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/ryogrid/SamehadaDB/lib/common" "github.com/ryogrid/SamehadaDB/lib/storage/buffer" + "github.com/ryogrid/SamehadaDB/lib/storage/index/index_common" "github.com/ryogrid/SamehadaDB/lib/storage/page" "github.com/ryogrid/SamehadaDB/lib/types" "unsafe" @@ -61,51 +62,19 @@ const ( offsetForward = offsetEntryCnt + sizeEntryCnt offsetFreeSpacePointer = offsetForward + sizeForward offsetEntryInfos = offsetFreeSpacePointer + sizeFreeSpacePointer - sizeEntryValue = uint32(8) ) -type SkipListPair struct { - Key types.Value - Value uint64 -} - type SkipListCornerInfo struct { PageId types.PageID UpdateCounter types.LSN } -func (sp SkipListPair) Serialize() []byte { - keyInBytes := sp.Key.Serialize() - valBuf := new(bytes.Buffer) - binary.Write(valBuf, binary.LittleEndian, sp.Value) - valInBytes := valBuf.Bytes() - - retBuf := new(bytes.Buffer) - retBuf.Write(keyInBytes) - retBuf.Write(valInBytes) - return retBuf.Bytes() -} - -func NewSkipListPairFromBytes(buf []byte, keyType types.TypeID) *SkipListPair { - dataLen := len(buf) - valPartOffset := dataLen - int(sizeEntryValue) - key := types.NewValueFromBytes(buf[:valPartOffset], keyType) - value := uint64(types.NewUInt64FromBytes(buf[valPartOffset:])) - return &SkipListPair{*key, value} -} - -func (sp SkipListPair) GetDataSize() uint32 { - keyInBytes := sp.Key.Serialize() - - return uint32(len(keyInBytes)) + sizeEntryValue -} - type SkipListBlockPage struct { page.Page } // ATTENTION: caller must call UnpinPage with appropriate dirty flag call to returned object -func NewSkipListBlockPage(bpm *buffer.BufferPoolManager, level int32, smallestListPair SkipListPair) *SkipListBlockPage { +func NewSkipListBlockPage(bpm *buffer.BufferPoolManager, level int32, smallestListPair index_common.IndexEntry) *SkipListBlockPage { page_ := bpm.NewPage() if page_ == nil { panic("NewPage can't allocate more page!") @@ -146,7 +115,7 @@ func (node *SkipListBlockPage) ValueAt(idx int32, keyType types.TypeID) uint64 { // if not found, returns info of nearest smaller key // binary search is used for search // https://www.cs.usfca.edu/~galles/visualization/Search.html -func (node *SkipListBlockPage) FindEntryByKey(key *types.Value) (found bool, entry *SkipListPair, index int32) { +func (node *SkipListBlockPage) FindEntryByKey(key *types.Value) (found bool, entry *index_common.IndexEntry, index int32) { if node.GetEntryCnt() == 1 { if node.GetEntry(0, key.ValueType()).Key.CompareEquals(*key) { return true, node.GetEntry(0, key.ValueType()), 0 @@ -221,7 +190,7 @@ func (node *SkipListBlockPage) updateEntryInfosAtInsert(idx int, dataSize uint32 // in contrast, new entry data is placed always tail of entries data area // idx==-1 -> data's inddx become 0 (insert to head of entries) // idx==entryCnt -> data's index become entryCnt (insert next of last entry) -func (node *SkipListBlockPage) InsertInner(idx int, slp *SkipListPair) { +func (node *SkipListBlockPage) InsertInner(idx int, slp *index_common.IndexEntry) { // data copy of slp arg to tail of entry data space // the tail is pointed by freeSpacePointer insertData := slp.Serialize() @@ -289,7 +258,7 @@ func (node *SkipListBlockPage) Insert(key *types.Value, value uint64, bpm *buffe // panic("key duplication is not supported yet!") //} - node.SetEntry(int(foundIdx), &SkipListPair{*key, value}) + node.SetEntry(int(foundIdx), &index_common.IndexEntry{*key, value}) //fmt.Printf("end of Insert of SkipListBlockPage called! : key=%d page.entryCnt=%d len(page.entries)=%d\n", key.ToInteger(), node.entryCnt, len(node.entries)) if node.GetEntry(int(foundIdx), key.ValueType()).Key.CompareEquals(*key) { @@ -309,7 +278,7 @@ func (node *SkipListBlockPage) Insert(key *types.Value, value uint64, bpm *buffe //fmt.Printf("Insert %v\n", key.ToIFValue()) //fmt.Printf("not found at Insert of SkipListBlockPage. foundIdx=%d\n", foundIdx) - if node.getFreeSpaceRemaining() < node.GetSpecifiedSLPNeedSpace(&SkipListPair{*key, value}) { + if node.getFreeSpaceRemaining() < node.GetSpecifiedSLPNeedSpace(&index_common.IndexEntry{*key, value}) { // this node is full. so node split is needed // first, split this node at center of entry list @@ -345,7 +314,7 @@ func (node *SkipListBlockPage) Insert(key *types.Value, value uint64, bpm *buffe // insert to new node newSmallerIdx := foundIdx - splitIdx - 1 - insEntry := &SkipListPair{*key, value} + insEntry := &index_common.IndexEntry{*key, value} if key.ValueType() == types.Varchar && (newNode.GetSpecifiedSLPNeedSpace(insEntry) > newNode.getFreeSpaceRemaining()) { panic("not enough space for insert (new node)") } @@ -364,7 +333,7 @@ func (node *SkipListBlockPage) Insert(key *types.Value, value uint64, bpm *buffe // foundIdx is index of nearlest smaller key entry // new entry is inserted next of the entry - insEntry := &SkipListPair{*key, value} + insEntry := &index_common.IndexEntry{*key, value} if key.ValueType() == types.Varchar && (node.GetSpecifiedSLPNeedSpace(insEntry) > node.getFreeSpaceRemaining()) { panic("not enough space for insert (parent node)") } @@ -390,7 +359,7 @@ func (node *SkipListBlockPage) Insert(key *types.Value, value uint64, bpm *buffe node.SetLSN(node.GetLSN() + 1) - insEntry := &SkipListPair{*key, value} + insEntry := &index_common.IndexEntry{*key, value} node.InsertInner(int(foundIdx), insEntry) bpm.UnpinPage(node.GetPageId(), true) @@ -662,7 +631,7 @@ func FindSLBPFromList(list []*SkipListBlockPage, pageID types.PageID) *SkipListB return nil } -func (node *SkipListBlockPage) newNodeAndUpdateChain(idx int32, bpm *buffer.BufferPoolManager, corners []SkipListCornerInfo, level int32, keyType types.TypeID, lockedAndPinnedNodes []*SkipListBlockPage, insertEntry *SkipListPair) *SkipListBlockPage { +func (node *SkipListBlockPage) newNodeAndUpdateChain(idx int32, bpm *buffer.BufferPoolManager, corners []SkipListCornerInfo, level int32, keyType types.TypeID, lockedAndPinnedNodes []*SkipListBlockPage, insertEntry *index_common.IndexEntry) *SkipListBlockPage { var newNode *SkipListBlockPage if insertEntry != nil { newNode = NewSkipListBlockPage(bpm, level, *insertEntry) @@ -752,9 +721,9 @@ func (node *SkipListBlockPage) SetEntryCnt(cnt int32) { copy(node.Data()[offsetEntryCnt:], cntInBytes) } -func (node *SkipListBlockPage) GetEntries(keyType types.TypeID) []*SkipListPair { +func (node *SkipListBlockPage) GetEntries(keyType types.TypeID) []*index_common.IndexEntry { entryNum := int(node.GetEntryCnt()) - retArr := make([]*SkipListPair, 0) + retArr := make([]*index_common.IndexEntry, 0) for ii := 0; ii < entryNum; ii++ { retArr = append(retArr, node.GetEntry(ii, keyType)) } @@ -805,17 +774,17 @@ func (node *SkipListBlockPage) SetFreeSpacePointer(pointOffset uint32) { copy(node.Data()[offset:], pointOffsetInBytes) } -func (node *SkipListBlockPage) GetEntry(idx int, keyType types.TypeID) *SkipListPair { +func (node *SkipListBlockPage) GetEntry(idx int, keyType types.TypeID) *index_common.IndexEntry { offset := node.GetEntryOffset(idx) entrySize := node.GetEntrySize(idx) - return NewSkipListPairFromBytes(node.Data()[offset:offset+entrySize], keyType) + return index_common.NewIndexEntryFromBytes(node.Data()[offset:offset+entrySize], keyType) } // ATTENTION: // this method can be called only when... // - it is guranteed that new entry insert doesn't cause overflow of node space capacity // - key of target entry doesn't exist in this node -func (node *SkipListBlockPage) SetEntry(idx int, entry *SkipListPair) { +func (node *SkipListBlockPage) SetEntry(idx int, entry *index_common.IndexEntry) { // at current design, // - duplicated key is not supported // - this SkipList is used for index of RDBMS, so update of value part (stores record data offset at DB file) @@ -832,7 +801,7 @@ func (node *SkipListBlockPage) SetEntry(idx int, entry *SkipListPair) { node.SetEntryCnt(node.GetEntryCnt() + 1) } -func (node *SkipListBlockPage) SetEntries(entries []*SkipListPair) { +func (node *SkipListBlockPage) SetEntries(entries []*index_common.IndexEntry) { buf := new(bytes.Buffer) entryNum := len(entries) // order of elements on buf becomes descending order in contrast with entries arg @@ -867,8 +836,8 @@ func (node *SkipListBlockPage) SetEntries(entries []*SkipListPair) { } // since header space grow with insertion entry, memory size which is needed -// for insertion is not same with size of SkipListPair object -func (node *SkipListBlockPage) GetSpecifiedSLPNeedSpace(slp *SkipListPair) uint32 { +// for insertion is not same with size of IndexEntry object +func (node *SkipListBlockPage) GetSpecifiedSLPNeedSpace(slp *index_common.IndexEntry) uint32 { return slp.GetDataSize() + sizeEntryInfo } diff --git a/lib/storage/page/skip_list_page/skip_list_header_page.go b/lib/storage/page/skip_list_page/skip_list_header_page.go index 263e8fa4..ad17990b 100644 --- a/lib/storage/page/skip_list_page/skip_list_header_page.go +++ b/lib/storage/page/skip_list_page/skip_list_header_page.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "github.com/ryogrid/SamehadaDB/lib/storage/buffer" + "github.com/ryogrid/SamehadaDB/lib/storage/index/index_common" "github.com/ryogrid/SamehadaDB/lib/storage/page" "github.com/ryogrid/SamehadaDB/lib/types" "math" @@ -43,33 +44,33 @@ func NewSkipListStartBlockPage(bpm *buffer.BufferPoolManager, keyType types.Type var startNode *SkipListBlockPage = nil switch keyType { case types.Integer: - startNode = NewSkipListBlockPage(bpm, MAX_FOWARD_LIST_LEN, SkipListPair{types.NewInteger(math.MinInt32), 0}) + startNode = NewSkipListBlockPage(bpm, MAX_FOWARD_LIST_LEN, index_common.IndexEntry{types.NewInteger(math.MinInt32), 0}) case types.Float: - startNode = NewSkipListBlockPage(bpm, MAX_FOWARD_LIST_LEN, SkipListPair{types.NewFloat(-1.0 * math.MaxFloat32), 0}) + startNode = NewSkipListBlockPage(bpm, MAX_FOWARD_LIST_LEN, index_common.IndexEntry{types.NewFloat(-1.0 * math.MaxFloat32), 0}) case types.Varchar: v := types.NewVarchar("") v.SetInfMin() - startNode = NewSkipListBlockPage(bpm, MAX_FOWARD_LIST_LEN, SkipListPair{v, 0}) + startNode = NewSkipListBlockPage(bpm, MAX_FOWARD_LIST_LEN, index_common.IndexEntry{v, 0}) case types.Boolean: - startNode = NewSkipListBlockPage(bpm, MAX_FOWARD_LIST_LEN, SkipListPair{types.NewBoolean(false), 0}) + startNode = NewSkipListBlockPage(bpm, MAX_FOWARD_LIST_LEN, index_common.IndexEntry{types.NewBoolean(false), 0}) } var sentinelNode *SkipListBlockPage = nil switch keyType { case types.Integer: - pl := SkipListPair{types.NewInteger(0), 0} + pl := index_common.IndexEntry{types.NewInteger(0), 0} pl.Key = *pl.Key.SetInfMax() sentinelNode = NewSkipListBlockPage(bpm, MAX_FOWARD_LIST_LEN, pl) case types.Float: - pl := SkipListPair{types.NewFloat(0), 0} + pl := index_common.IndexEntry{types.NewFloat(0), 0} pl.Key = *pl.Key.SetInfMax() sentinelNode = NewSkipListBlockPage(bpm, MAX_FOWARD_LIST_LEN, pl) case types.Varchar: - pl := SkipListPair{types.NewVarchar(""), 0} + pl := index_common.IndexEntry{types.NewVarchar(""), 0} pl.Key = *pl.Key.SetInfMax() sentinelNode = NewSkipListBlockPage(bpm, MAX_FOWARD_LIST_LEN, pl) case types.Boolean: - pl := SkipListPair{types.NewBoolean(false), 0} + pl := index_common.IndexEntry{types.NewBoolean(false), 0} pl.Key = *pl.Key.SetInfMax() sentinelNode = NewSkipListBlockPage(bpm, MAX_FOWARD_LIST_LEN, pl) } diff --git a/lib/types/column_value.go b/lib/types/column_value.go index e6a46e8b..fb2c0d58 100644 --- a/lib/types/column_value.go +++ b/lib/types/column_value.go @@ -332,6 +332,27 @@ func (v Value) Serialize() []byte { return []byte{} } +// no length info and isNull info +func (v Value) SerializeOnlyVal() []byte { + switch v.valueType { + case Integer: + buf := new(bytes.Buffer) + binary.Write(buf, binary.LittleEndian, v.ToInteger()) + return buf.Bytes() + case Float: + buf := new(bytes.Buffer) + binary.Write(buf, binary.LittleEndian, v.ToFloat()) + return buf.Bytes() + case Varchar: + return []byte(v.ToVarchar()) + case Boolean: + buf := new(bytes.Buffer) + binary.Write(buf, binary.LittleEndian, v.ToBoolean()) + return buf.Bytes() + } + return []byte{} +} + // Size returns the size in bytes that the type will occupy inside the tuple func (v Value) Size() uint32 { // all type occupies the whether NULL or not + 1 byte for the info storage diff --git a/server/go.mod b/server/go.mod index 4e99185e..1a8ab91a 100644 --- a/server/go.mod +++ b/server/go.mod @@ -8,7 +8,7 @@ toolchain go1.21.2 require ( github.com/ant0ine/go-json-rest v3.3.2+incompatible - github.com/ryogrid/SamehadaDB/lib v0.0.0-00010101000000-000000000000 + github.com/ryogrid/SamehadaDB/lib v0.0.0-20240725021953-263ea0c5c011 github.com/vmihailenco/msgpack/v5 v5.4.1 ) @@ -30,6 +30,7 @@ require ( github.com/pingcap/tidb v1.1.0-beta.0.20200630082100-328b6d0a955c // indirect github.com/pingcap/tipb v0.0.0-20200522051215-f31a15d98fce // indirect github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect + github.com/ryogrid/bltree-go-for-embedding v1.0.4 // indirect github.com/shirou/gopsutil v2.19.10+incompatible // indirect github.com/sirupsen/logrus v1.6.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/server/go.sum b/server/go.sum index bbb25f33..26e37a44 100644 --- a/server/go.sum +++ b/server/go.sum @@ -450,6 +450,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryogrid/bltree-go-for-embedding v1.0.4 h1:LAvotdhSovWV3T2m8X9LtcKt7E+NrndEcLgUCunHwLg= +github.com/ryogrid/bltree-go-for-embedding v1.0.4/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=