Skip to content

Commit

Permalink
fixed BTreeIndex (#49)
Browse files Browse the repository at this point in the history 
* parallel TestBLTree_deleteManyConcurrently2 passed but TestKeyDuplicateBTreePrallelTxnStrideInteger fails.

* TestKeyDuplicateInsertDeleteWithBTreeIndexInteger passed without no risriction.

* TestKeyDuplicateInsertDeleteWithBTreeIndexVarchar passed(50 length fixed. SERIAL EXEC, 800stride, 3000op).

* TestKeyDuplicateInsertDeleteWithBTreeIndexVarchar passed(50 based variable length. SERIAL EXEC, 800stride, 3000op).

* WIP

* WIP2

* TestKeyDuplicateBTreePrallelTxnStrideInteger (800stride, 3000op, parallel) almost passed.

* TestKeyDuplicateBTreePrallelTxnStrideVarchar (800stride, 300op, parallel) almost passed.

* TestKeyDuplicateBTreePrallelTxnStrideVarchar (800stride, 30000op, parallel) almost passed.

* TestKeyDuplicateBTreePrallelTxnStrideVarchar (800stride, 30000op, parallel, update only exclusive, 50char fixed, range scan case 1-3 only) passed once.

* Integer and Varchar testcase passed. Varchar(all writes get giantlock, 50bytes fixed size key)

* Integer and Varchar testcase passed. Varchar(all writes and reads get giant exclusive lock, 50bytes fixed size key)

* TestKeyDuplicateBTreePrallelTxnStrideVarchar(no strange giant lock, fixed length) really passed

* TestKeyDuplicateBTreePrallelTxnStrideVarchar(no strange giant lock, max 10charctors, internally 50bytes fixed) passed

* TestKeyDuplicateBTreePrallelTxnStrideVarchar(no strange giant lock, max 10charctors, 10000op, internally 50bytes fixed) passed in not long time.

* updated bltree-go-for-embedding lib.
  • Loading branch information
@ryogrid
ryogrid authored Sep 12, 2024
1 parent 560ffda commit e3d8142
Show file tree
Hide file tree
Showing 7 changed files with 64 additions and 24 deletions.
17 changes: 11 additions & 6 deletions lib/execution/executors/executor_test/btree_index_executor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,21 +125,26 @@ func testBTreeParallelTxnStrideRoot[T int32 | float32 | string](t *testing.T, ke

//InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 3000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20)

InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 30000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20)
//InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 30000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20)

//InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 1000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, SERIAL_EXEC, 1)
//InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 30000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, SERIAL_EXEC, 1)

//InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 3000, 13, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, PARALLEL_EXEC, 20)
//InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 30000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20)
InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 200, 10000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20)
case types.Float:
InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 240, 1000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20)
//InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 240, 1000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20)
InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 200, 10000, 13, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20)
case types.Varchar:
//InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 5000, 17, 0, bpoolSize, index_constants.INDEX_KIND_SKIP_LIST, PARALLEL_EXEC, 20)
//InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 5000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20)
//InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 1000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, SERIAL_EXEC, 1)
//InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 5000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, SERIAL_EXEC, 1)

//InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 400, 3000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20)
InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 30000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20)
//InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 3000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, SERIAL_EXEC, 1)
//InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 30000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20)
//InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 200, 10000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20)
InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 10000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, PARALLEL_EXEC, 20)
//InnerTestParallelTxnsQueryingIndexUsedColumns[T](t, keyType, 800, 30000, 17, 0, bpoolSize, index_constants.INDEX_KIND_BTREE, SERIAL_EXEC, 1)
default:
panic("not implemented!")
}
Expand Down
2 changes: 1 addition & 1 deletion lib/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ require (
github.com/notEpsilon/go-pair v0.0.0-20221220200415-e91ef28c6c0b
github.com/pingcap/parser v0.0.0-20200623164729-3a18f1e5dceb
github.com/pingcap/tidb v1.1.0-beta.0.20200630082100-328b6d0a955c
github.com/ryogrid/bltree-go-for-embedding v1.0.10
github.com/ryogrid/bltree-go-for-embedding v1.0.11
github.com/spaolacci/murmur3 v1.1.0
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
)
Expand Down
4 changes: 2 additions & 2 deletions lib/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -448,8 +448,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryogrid/bltree-go-for-embedding v1.0.10 h1:W+0yrVJ25+cWKwV+7bLDq+p2w93o4NI1NGa8qKJDHNQ=
github.com/ryogrid/bltree-go-for-embedding v1.0.10/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE=
github.com/ryogrid/bltree-go-for-embedding v1.0.11 h1:99yiTJXFka8O+eP3XjPhBaNwq6MqonZE4C3A0nIPEuY=
github.com/ryogrid/bltree-go-for-embedding v1.0.11/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE=
github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
Expand Down
20 changes: 18 additions & 2 deletions lib/samehada/samehada_util/samehada_util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,10 +177,9 @@ func GetRandomPrimitiveVal[T int32 | float32 | string](keyType types.TypeID, max
return ret.(T)
case types.Varchar:
//var ret interface{} = *GetRandomStr(200)
// TODO: (SDB) FOR DEBUG: length is fixed
//var ret interface{} = *GetRandomStr(50, false)
//var ret interface{} = *GetRandomStr(50, true)
var ret interface{} = *GetRandomStr(10, true)
var ret interface{} = *GetRandomStr(10, false)
return ret.(T)
default:
panic("not supported keyType")
Expand Down Expand Up @@ -522,3 +521,20 @@ func PrintExecuteResults(results [][]*types.Value) {
fmt.Println("")
}
}

func FillZeroValues(key []byte, maxKeyLen int) []byte {
if len(key) > maxKeyLen-12-2 {
panic("key length is too long")
}
ret := append(key, make([]byte, maxKeyLen-len(key)-2)...)
paddingLen := uint16(maxKeyLen - len(key) - 2)
paddingLenBytes := make([]byte, 2)
binary.BigEndian.PutUint16(paddingLenBytes, paddingLen)
ret = append(ret, paddingLenBytes...)
return ret
}

func EliminateZeroValues(key []byte) []byte {
zeroPaddingLen := binary.BigEndian.Uint16(key[len(key)-2:])
return key[:len(key)-int(zeroPaddingLen)-2]
}
39 changes: 29 additions & 10 deletions lib/storage/index/btree_index.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ import (
"sync"
)

const MaxKeyLen = 50

type BtreeIndexIterator struct {
itr *blink_tree.BLTreeItr
valType types.TypeID
Expand Down Expand Up @@ -54,6 +56,7 @@ func (btreeItr *BtreeIndexIterator) Next() (done bool, err error, key *types.Val
newKeyBytes = append(newKeyBytes, keyLenBuf...)
newKeyBytes = append(newKeyBytes, keyBytes...)
case types.Varchar:
keyBytes = samehada_util.EliminateZeroValues(keyBytes)
// 4 is {0, 0, 0, 0}
// 8 is length of packedRID
newKeyBytes = keyBytes[:len(keyBytes)-4-8]
Expand All @@ -65,6 +68,9 @@ func (btreeItr *BtreeIndexIterator) Next() (done bool, err error, key *types.Val
return false, nil, decodedKey, &unpackedRID
}

// Attention: when column type is Varchar, key length should be less than (MaxKeyLen - 12 - 2)
//
// 12 is {0, 0, 0, 0} and 8bytes of packedRID, 2byte is for length of zero padding
type BTreeIndex struct {
container *blink_tree.BLTree
metadata *IndexMetadata
Expand Down Expand Up @@ -100,8 +106,8 @@ func (btidx *BTreeIndex) insertEntryInner(key *tuple.Tuple, rid page.RID, txn in
convedKeyVal := samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgKeyVal, &rid)

if isNoLock == false {
btidx.rwMtx.Lock()
defer btidx.rwMtx.Unlock()
btidx.rwMtx.RLock()
defer btidx.rwMtx.RUnlock()
}

ridBytes := samehada_util.PackRIDto8bytes(&rid)
Expand All @@ -110,7 +116,13 @@ func (btidx *BTreeIndex) insertEntryInner(key *tuple.Tuple, rid page.RID, txn in
//sixBytesVal := [6]byte{valBuf[0], valBuf[1], valBuf[2], valBuf[3], valBuf[6], valBuf[7]}
sixBytesVal := [6]byte{ridBytes[0], ridBytes[1], ridBytes[2], ridBytes[3], ridBytes[6], ridBytes[7]}

btidx.container.InsertKey(convedKeyVal.SerializeOnlyVal(), 0, sixBytesVal, true)
keyBytes := convedKeyVal.SerializeOnlyVal()
if orgKeyVal.ValueType() == types.Varchar {
// fill 0 to make keyBytes length to MaxKeyLen for avoiding bug of BLTree
keyBytes = samehada_util.FillZeroValues(keyBytes, MaxKeyLen)
}

btidx.container.InsertKey(keyBytes, 0, sixBytesVal, true)
}

func (btidx *BTreeIndex) InsertEntry(key *tuple.Tuple, rid page.RID, txn interface{}) {
Expand All @@ -124,10 +136,16 @@ func (btidx *BTreeIndex) deleteEntryInner(key *tuple.Tuple, rid page.RID, txn in
convedKeyVal := samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgKeyVal, &rid)

if isNoLock == false {
btidx.rwMtx.Lock()
defer btidx.rwMtx.Unlock()
btidx.rwMtx.RLock()
defer btidx.rwMtx.RUnlock()
}
btidx.container.DeleteKey(convedKeyVal.SerializeOnlyVal(), 0)
keyBytes := convedKeyVal.SerializeOnlyVal()
if orgKeyVal.ValueType() == types.Varchar {
// fill 0 to make keyBytes length to MaxKeyLen for avoiding bug of BLTree
keyBytes = samehada_util.FillZeroValues(keyBytes, MaxKeyLen)
}

btidx.container.DeleteKey(keyBytes, 0)
}

func (btidx *BTreeIndex) DeleteEntry(key *tuple.Tuple, rid page.RID, txn interface{}) {
Expand All @@ -140,7 +158,7 @@ func (btidx *BTreeIndex) ScanKey(key *tuple.Tuple, txn interface{}) []page.RID {
smallestKeyVal := samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgKeyVal, &page.RID{0, 0})
biggestKeyVal := samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgKeyVal, &page.RID{math.MaxInt32, math.MaxUint32})

//btidx.rwMtx.RLock()
btidx.rwMtx.RLock()
// Attention: returned itr's containing keys are string type Value which is constructed with byte arr of concatenated original key and value
rangeItr := btidx.container.GetRangeItr(smallestKeyVal.SerializeOnlyVal(), biggestKeyVal.SerializeOnlyVal())

Expand All @@ -152,7 +170,7 @@ func (btidx *BTreeIndex) ScanKey(key *tuple.Tuple, txn interface{}) []page.RID {
//uintRID := binary.BigEndian.Uint64(eightBytesRID[:])
retArr = append(retArr, samehada_util.Unpack8BytesToRID(eightBytesRID[:]))
}
//btidx.rwMtx.RUnlock()
btidx.rwMtx.RUnlock()

return retArr
}
Expand Down Expand Up @@ -182,8 +200,9 @@ func (btidx *BTreeIndex) GetRangeScanIterator(start_key *tuple.Tuple, end_key *t
biggestKeyVal = samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgEndKeyVal, &page.RID{math.MaxInt32, math.MaxUint32})
}

//btidx.rwMtx.RLock()
//defer btidx.rwMtx.RUnlock()
btidx.rwMtx.RLock()
defer btidx.rwMtx.RUnlock()

var smalledKeyBytes []byte
var biggestKeyBytes []byte

Expand Down
2 changes: 1 addition & 1 deletion server/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ require (
github.com/pingcap/tidb v1.1.0-beta.0.20200630082100-328b6d0a955c // indirect
github.com/pingcap/tipb v0.0.0-20200522051215-f31a15d98fce // indirect
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect
github.com/ryogrid/bltree-go-for-embedding v1.0.10 // indirect
github.com/ryogrid/bltree-go-for-embedding v1.0.11 // indirect
github.com/shirou/gopsutil v2.19.10+incompatible // indirect
github.com/sirupsen/logrus v1.6.0 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions server/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -450,8 +450,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryogrid/bltree-go-for-embedding v1.0.10 h1:W+0yrVJ25+cWKwV+7bLDq+p2w93o4NI1NGa8qKJDHNQ=
github.com/ryogrid/bltree-go-for-embedding v1.0.10/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE=
github.com/ryogrid/bltree-go-for-embedding v1.0.11 h1:99yiTJXFka8O+eP3XjPhBaNwq6MqonZE4C3A0nIPEuY=
github.com/ryogrid/bltree-go-for-embedding v1.0.11/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE=
github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
Expand Down

0 comments on commit e3d8142

Please sign in to comment.