From e3d814274c301242b1d5b4a40c8fe4857db75ad9 Mon Sep 17 00:00:00 2001 From: Ryo Kanbayashi Date: Fri, 13 Sep 2024 00:17:38 +0900 Subject: [PATCH] fixed BTreeIndex (#49) * parallel TestBLTree_deleteManyConcurrently2 passed but TestKeyDuplicateBTreePrallelTxnStrideInteger fails. * TestKeyDuplicateInsertDeleteWithBTreeIndexInteger passed without no risriction. * TestKeyDuplicateInsertDeleteWithBTreeIndexVarchar passed(50 length fixed. SERIAL EXEC, 800stride, 3000op). * TestKeyDuplicateInsertDeleteWithBTreeIndexVarchar passed(50 based variable length. SERIAL EXEC, 800stride, 3000op). * WIP * WIP2 * TestKeyDuplicateBTreePrallelTxnStrideInteger (800stride, 3000op, parallel) almost passed. * TestKeyDuplicateBTreePrallelTxnStrideVarchar (800stride, 300op, parallel) almost passed. * TestKeyDuplicateBTreePrallelTxnStrideVarchar (800stride, 30000op, parallel) almost passed. * TestKeyDuplicateBTreePrallelTxnStrideVarchar (800stride, 30000op, parallel, update only exclusive, 50char fixed, range scan case 1-3 only) passed once. * Integer and Varchar testcase passed. Varchar(all writes get giantlock, 50bytes fixed size key) * Integer and Varchar testcase passed. Varchar(all writes and reads get giant exclusive lock, 50bytes fixed size key) * TestKeyDuplicateBTreePrallelTxnStrideVarchar(no strange giant lock, fixed length) really passed * TestKeyDuplicateBTreePrallelTxnStrideVarchar(no strange giant lock, max 10charctors, internally 50bytes fixed) passed * TestKeyDuplicateBTreePrallelTxnStrideVarchar(no strange giant lock, max 10charctors, 10000op, internally 50bytes fixed) passed in not long time. * updated bltree-go-for-embedding lib. --- .../btree_index_executor_test.go | 17 +++++--- lib/go.mod | 2 +- lib/go.sum | 4 +- lib/samehada/samehada_util/samehada_util.go | 20 +++++++++- lib/storage/index/btree_index.go | 39 ++++++++++++++----- server/go.mod | 2 +- server/go.sum | 4 +- 7 files changed, 64 insertions(+), 24 deletions(-) diff --git a/lib/execution/executors/executor_test/btree_index_executor_test.go b/lib/execution/executors/executor_test/btree_index_executor_test.go index 5c4d0b8e..ff728d26 100644 --- a/lib/execution/executors/executor_test/btree_index_executor_test.go +++ b/lib/execution/executors/executor_test/btree_index_executor_test.go @@ -125,21 +125,26 @@ func testBTreeParallelTxnStrideRoot[T int32 | float32 | string](t *testing.T, ke //InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 3000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20) - InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 30000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20) + //InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 30000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20) - //InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 1000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, SERIAL_EXEC, 1) + //InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 30000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, SERIAL_EXEC, 1) - //InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 3000, 13, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, PARALLEL_EXEC, 20) + //InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 30000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20) + InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 200, 10000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20) case types.Float: - InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 240, 1000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20) + //InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 240, 1000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20) + InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 200, 10000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20) case types.Varchar: //InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 5000, 17, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, PARALLEL_EXEC, 20) //InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 5000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20) //InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 1000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, SERIAL_EXEC, 1) //InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 5000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, SERIAL_EXEC, 1) - //InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 3000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20) - InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 30000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20) + //InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 3000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, SERIAL_EXEC, 1) + //InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 30000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20) + //InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 200, 10000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20) + InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 10000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20) + //InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 30000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, SERIAL_EXEC, 1) default: panic("not implemented!") } diff --git a/lib/go.mod b/lib/go.mod index ebd35934..b535bf0a 100644 --- a/lib/go.mod +++ b/lib/go.mod @@ -14,7 +14,7 @@ require ( github.com/notEpsilon/go-pair v0.0.0-20221220200415-e91ef28c6c0b github.com/pingcap/parser v0.0.0-20200623164729-3a18f1e5dceb github.com/pingcap/tidb v1.1.0-beta.0.20200630082100-328b6d0a955c - github.com/ryogrid/bltree-go-for-embedding v1.0.10 + github.com/ryogrid/bltree-go-for-embedding v1.0.11 github.com/spaolacci/murmur3 v1.1.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 ) diff --git a/lib/go.sum b/lib/go.sum index 91f97fb6..992f8688 100644 --- a/lib/go.sum +++ b/lib/go.sum @@ -448,8 +448,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryogrid/bltree-go-for-embedding v1.0.10 h1:W+0yrVJ25+cWKwV+7bLDq+p2w93o4NI1NGa8qKJDHNQ= -github.com/ryogrid/bltree-go-for-embedding v1.0.10/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE= +github.com/ryogrid/bltree-go-for-embedding v1.0.11 h1:99yiTJXFka8O+eP3XjPhBaNwq6MqonZE4C3A0nIPEuY= +github.com/ryogrid/bltree-go-for-embedding v1.0.11/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= diff --git a/lib/samehada/samehada_util/samehada_util.go b/lib/samehada/samehada_util/samehada_util.go index 29d746cc..6d147718 100644 --- a/lib/samehada/samehada_util/samehada_util.go +++ b/lib/samehada/samehada_util/samehada_util.go @@ -177,10 +177,9 @@ func GetRandomPrimitiveVal[T int32 | float32 | string](keyType types.TypeID, max return ret.(T) case types.Varchar: //var ret interface{} = *GetRandomStr(200) - // TODO: (SDB) FOR DEBUG: length is fixed //var ret interface{} = *GetRandomStr(50, false) //var ret interface{} = *GetRandomStr(50, true) - var ret interface{} = *GetRandomStr(10, true) + var ret interface{} = *GetRandomStr(10, false) return ret.(T) default: panic("not supported keyType") @@ -522,3 +521,20 @@ func PrintExecuteResults(results [][]*types.Value) { fmt.Println("") } } + +func FillZeroValues(key []byte, maxKeyLen int) []byte { + if len(key) > maxKeyLen-12-2 { + panic("key length is too long") + } + ret := append(key, make([]byte, maxKeyLen-len(key)-2)...) + paddingLen := uint16(maxKeyLen - len(key) - 2) + paddingLenBytes := make([]byte, 2) + binary.BigEndian.PutUint16(paddingLenBytes, paddingLen) + ret = append(ret, paddingLenBytes...) + return ret +} + +func EliminateZeroValues(key []byte) []byte { + zeroPaddingLen := binary.BigEndian.Uint16(key[len(key)-2:]) + return key[:len(key)-int(zeroPaddingLen)-2] +} diff --git a/lib/storage/index/btree_index.go b/lib/storage/index/btree_index.go index c9328352..da3cc440 100644 --- a/lib/storage/index/btree_index.go +++ b/lib/storage/index/btree_index.go @@ -16,6 +16,8 @@ import ( "sync" ) +const MaxKeyLen = 50 + type BtreeIndexIterator struct { itr *blink_tree.BLTreeItr valType types.TypeID @@ -54,6 +56,7 @@ func (btreeItr *BtreeIndexIterator) Next() (done bool, err error, key *types.Val newKeyBytes = append(newKeyBytes, keyLenBuf...) newKeyBytes = append(newKeyBytes, keyBytes...) case types.Varchar: + keyBytes = samehada_util.EliminateZeroValues(keyBytes) // 4 is {0, 0, 0, 0} // 8 is length of packedRID newKeyBytes = keyBytes[:len(keyBytes)-4-8] @@ -65,6 +68,9 @@ func (btreeItr *BtreeIndexIterator) Next() (done bool, err error, key *types.Val return false, nil, decodedKey, &unpackedRID } +// Attention: when column type is Varchar, key length should be less than (MaxKeyLen - 12 - 2) +// +// 12 is {0, 0, 0, 0} and 8bytes of packedRID, 2byte is for length of zero padding type BTreeIndex struct { container *blink_tree.BLTree metadata *IndexMetadata @@ -100,8 +106,8 @@ func (btidx *BTreeIndex) insertEntryInner(key *tuple.Tuple, rid page.RID, txn in convedKeyVal := samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgKeyVal, &rid) if isNoLock == false { - btidx.rwMtx.Lock() - defer btidx.rwMtx.Unlock() + btidx.rwMtx.RLock() + defer btidx.rwMtx.RUnlock() } ridBytes := samehada_util.PackRIDto8bytes(&rid) @@ -110,7 +116,13 @@ func (btidx *BTreeIndex) insertEntryInner(key *tuple.Tuple, rid page.RID, txn in //sixBytesVal := [6]byte{valBuf[0], valBuf[1], valBuf[2], valBuf[3], valBuf[6], valBuf[7]} sixBytesVal := [6]byte{ridBytes[0], ridBytes[1], ridBytes[2], ridBytes[3], ridBytes[6], ridBytes[7]} - btidx.container.InsertKey(convedKeyVal.SerializeOnlyVal(), 0, sixBytesVal, true) + keyBytes := convedKeyVal.SerializeOnlyVal() + if orgKeyVal.ValueType() == types.Varchar { + // fill 0 to make keyBytes length to MaxKeyLen for avoiding bug of BLTree + keyBytes = samehada_util.FillZeroValues(keyBytes, MaxKeyLen) + } + + btidx.container.InsertKey(keyBytes, 0, sixBytesVal, true) } func (btidx *BTreeIndex) InsertEntry(key *tuple.Tuple, rid page.RID, txn interface{}) { @@ -124,10 +136,16 @@ func (btidx *BTreeIndex) deleteEntryInner(key *tuple.Tuple, rid page.RID, txn in convedKeyVal := samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgKeyVal, &rid) if isNoLock == false { - btidx.rwMtx.Lock() - defer btidx.rwMtx.Unlock() + btidx.rwMtx.RLock() + defer btidx.rwMtx.RUnlock() } - btidx.container.DeleteKey(convedKeyVal.SerializeOnlyVal(), 0) + keyBytes := convedKeyVal.SerializeOnlyVal() + if orgKeyVal.ValueType() == types.Varchar { + // fill 0 to make keyBytes length to MaxKeyLen for avoiding bug of BLTree + keyBytes = samehada_util.FillZeroValues(keyBytes, MaxKeyLen) + } + + btidx.container.DeleteKey(keyBytes, 0) } func (btidx *BTreeIndex) DeleteEntry(key *tuple.Tuple, rid page.RID, txn interface{}) { @@ -140,7 +158,7 @@ func (btidx *BTreeIndex) ScanKey(key *tuple.Tuple, txn interface{}) []page.RID { smallestKeyVal := samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgKeyVal, &page.RID{0, 0}) biggestKeyVal := samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgKeyVal, &page.RID{math.MaxInt32, math.MaxUint32}) - //btidx.rwMtx.RLock() + btidx.rwMtx.RLock() // Attention: returned itr's containing keys are string type Value which is constructed with byte arr of concatenated original key and value rangeItr := btidx.container.GetRangeItr(smallestKeyVal.SerializeOnlyVal(), biggestKeyVal.SerializeOnlyVal()) @@ -152,7 +170,7 @@ func (btidx *BTreeIndex) ScanKey(key *tuple.Tuple, txn interface{}) []page.RID { //uintRID := binary.BigEndian.Uint64(eightBytesRID[:]) retArr = append(retArr, samehada_util.Unpack8BytesToRID(eightBytesRID[:])) } - //btidx.rwMtx.RUnlock() + btidx.rwMtx.RUnlock() return retArr } @@ -182,8 +200,9 @@ func (btidx *BTreeIndex) GetRangeScanIterator(start_key *tuple.Tuple, end_key *t biggestKeyVal = samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgEndKeyVal, &page.RID{math.MaxInt32, math.MaxUint32}) } - //btidx.rwMtx.RLock() - //defer btidx.rwMtx.RUnlock() + btidx.rwMtx.RLock() + defer btidx.rwMtx.RUnlock() + var smalledKeyBytes []byte var biggestKeyBytes []byte diff --git a/server/go.mod b/server/go.mod index ce9bcebe..3420ce02 100644 --- a/server/go.mod +++ b/server/go.mod @@ -32,7 +32,7 @@ require ( github.com/pingcap/tidb v1.1.0-beta.0.20200630082100-328b6d0a955c // indirect github.com/pingcap/tipb v0.0.0-20200522051215-f31a15d98fce // indirect github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect - github.com/ryogrid/bltree-go-for-embedding v1.0.10 // indirect + github.com/ryogrid/bltree-go-for-embedding v1.0.11 // indirect github.com/shirou/gopsutil v2.19.10+incompatible // indirect github.com/sirupsen/logrus v1.6.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/server/go.sum b/server/go.sum index 3b227e08..6ee70103 100644 --- a/server/go.sum +++ b/server/go.sum @@ -450,8 +450,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryogrid/bltree-go-for-embedding v1.0.10 h1:W+0yrVJ25+cWKwV+7bLDq+p2w93o4NI1NGa8qKJDHNQ= -github.com/ryogrid/bltree-go-for-embedding v1.0.10/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE= +github.com/ryogrid/bltree-go-for-embedding v1.0.11 h1:99yiTJXFka8O+eP3XjPhBaNwq6MqonZE4C3A0nIPEuY= +github.com/ryogrid/bltree-go-for-embedding v1.0.11/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=