github.com/nutsdb/nutsdb@v1.0.4/sorted_set.go (about) 1 // Copyright 2023 The nutsdb Author. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package nutsdb 16 17 import ( 18 "bytes" 19 "errors" 20 "math/rand" 21 ) 22 23 var ( 24 ErrSortedSetNotFound = errors.New("the sortedSet does not exist") 25 26 ErrSortedSetMemberNotExist = errors.New("the member of sortedSet does not exist") 27 28 ErrSortedSetIsEmpty = errors.New("the sortedSet if empty") 29 ) 30 31 const ( 32 // SkipListMaxLevel represents the skipList max level number. 33 SkipListMaxLevel = 32 34 35 // SkipListP represents the p parameter of the skipList. 36 SkipListP = 0.25 37 ) 38 39 type SortedSet struct { 40 db *DB 41 M map[string]*SkipList 42 } 43 44 func NewSortedSet(db *DB) *SortedSet { 45 return &SortedSet{ 46 db: db, 47 M: map[string]*SkipList{}, 48 } 49 } 50 51 func (z *SortedSet) ZAdd(key string, score SCORE, value []byte, record *Record) error { 52 sortedSet, ok := z.M[key] 53 if !ok { 54 z.M[key] = newSkipList(z.db) 55 sortedSet = z.M[key] 56 } 57 58 return sortedSet.Put(score, value, record) 59 } 60 61 func (z *SortedSet) ZMembers(key string) (map[*Record]SCORE, error) { 62 sortedSet, ok := z.M[key] 63 64 if !ok { 65 return nil, ErrSortedSetNotFound 66 } 67 68 nodes := sortedSet.dict 69 70 members := make(map[*Record]SCORE, len(nodes)) 71 for _, node := range nodes { 72 members[node.record] = node.score 73 } 74 75 return members, nil 76 } 77 78 func (z *SortedSet) ZCard(key string) (int, error) { 79 if sortedSet, ok := z.M[key]; ok { 80 return int(sortedSet.length), nil 81 } 82 83 return 0, ErrSortedSetNotFound 84 } 85 86 func (z *SortedSet) ZCount(key string, start SCORE, end SCORE, opts *GetByScoreRangeOptions) (int, error) { 87 if sortedSet, ok := z.M[key]; ok { 88 return len(sortedSet.GetByScoreRange(start, end, opts)), nil 89 } 90 return 0, ErrSortedSetNotFound 91 } 92 93 func (z *SortedSet) ZPeekMax(key string) (*Record, SCORE, error) { 94 if sortedSet, ok := z.M[key]; ok { 95 node := sortedSet.PeekMax() 96 if node != nil { 97 return node.record, node.score, nil 98 } 99 return nil, 0, ErrSortedSetIsEmpty 100 } 101 102 return nil, 0, ErrSortedSetNotFound 103 } 104 105 func (z *SortedSet) ZPopMax(key string) (*Record, SCORE, error) { 106 if sortedSet, ok := z.M[key]; ok { 107 node := sortedSet.PopMax() 108 if node != nil { 109 return node.record, node.score, nil 110 } 111 return nil, 0, ErrSortedSetIsEmpty 112 } 113 114 return nil, 0, ErrSortedSetNotFound 115 } 116 117 func (z *SortedSet) ZPeekMin(key string) (*Record, SCORE, error) { 118 if sortedSet, ok := z.M[key]; ok { 119 node := sortedSet.PeekMin() 120 if node != nil { 121 return node.record, node.score, nil 122 } 123 return nil, 0, ErrSortedSetIsEmpty 124 } 125 126 return nil, 0, ErrSortedSetNotFound 127 } 128 129 func (z *SortedSet) ZPopMin(key string) (*Record, SCORE, error) { 130 if sortedSet, ok := z.M[key]; ok { 131 node := sortedSet.PopMin() 132 if node != nil { 133 return node.record, node.score, nil 134 } 135 return nil, 0, ErrSortedSetIsEmpty 136 } 137 138 return nil, 0, ErrSortedSetNotFound 139 } 140 141 func (z *SortedSet) ZRangeByScore(key string, start SCORE, end SCORE, opts *GetByScoreRangeOptions) ([]*Record, []float64, error) { 142 if sortedSet, ok := z.M[key]; ok { 143 144 nodes := sortedSet.GetByScoreRange(start, end, opts) 145 146 records := make([]*Record, len(nodes)) 147 scores := make([]float64, len(nodes)) 148 149 for i, node := range nodes { 150 records[i] = node.record 151 scores[i] = float64(node.score) 152 } 153 154 return records, scores, nil 155 } 156 157 return nil, nil, ErrSortedSetNotFound 158 } 159 160 func (z *SortedSet) ZRangeByRank(key string, start int, end int) ([]*Record, []float64, error) { 161 if sortedSet, ok := z.M[key]; ok { 162 163 nodes := sortedSet.GetByRankRange(start, end, false) 164 165 records := make([]*Record, len(nodes)) 166 scores := make([]float64, len(nodes)) 167 168 for i, node := range nodes { 169 records[i] = node.record 170 scores[i] = float64(node.score) 171 } 172 173 return records, scores, nil 174 } 175 176 return nil, nil, ErrSortedSetNotFound 177 } 178 179 func (z *SortedSet) ZRem(key string, value []byte) (*Record, error) { 180 if sortedSet, ok := z.M[key]; ok { 181 hash, err := getFnv32(value) 182 if err != nil { 183 return nil, err 184 } 185 node := sortedSet.Remove(hash) 186 if node != nil { 187 return node.record, nil 188 } 189 return nil, ErrSortedSetMemberNotExist 190 } 191 192 return nil, ErrSortedSetNotFound 193 } 194 195 func (z *SortedSet) ZRemRangeByRank(key string, start int, end int) error { 196 if sortedSet, ok := z.M[key]; ok { 197 198 _ = sortedSet.GetByRankRange(start, end, true) 199 return nil 200 } 201 202 return ErrSortedSetNotFound 203 } 204 205 func (z *SortedSet) getZRemRangeByRankNodes(key string, start int, end int) ([]*SkipListNode, error) { 206 if sortedSet, ok := z.M[key]; ok { 207 return sortedSet.GetByRankRange(start, end, false), nil 208 } 209 210 return []*SkipListNode{}, nil 211 } 212 213 func (z *SortedSet) ZRank(key string, value []byte) (int, error) { 214 if sortedSet, ok := z.M[key]; ok { 215 hash, err := getFnv32(value) 216 if err != nil { 217 return 0, err 218 } 219 rank := sortedSet.FindRank(hash) 220 if rank == 0 { 221 return 0, ErrSortedSetMemberNotExist 222 } 223 return rank, nil 224 } 225 return 0, ErrSortedSetNotFound 226 } 227 228 func (z *SortedSet) ZRevRank(key string, value []byte) (int, error) { 229 if sortedSet, ok := z.M[key]; ok { 230 hash, err := getFnv32(value) 231 if err != nil { 232 return 0, err 233 } 234 rank := sortedSet.FindRevRank(hash) 235 if rank == 0 { 236 return 0, ErrSortedSetMemberNotExist 237 } 238 return rank, nil 239 } 240 return 0, ErrSortedSetNotFound 241 } 242 243 func (z *SortedSet) ZScore(key string, value []byte) (float64, error) { 244 if sortedSet, ok := z.M[key]; ok { 245 node := sortedSet.GetByValue(value) 246 if node != nil { 247 return float64(sortedSet.GetByValue(value).score), nil 248 } 249 return 0, ErrSortedSetMemberNotExist 250 } 251 return 0, ErrSortedSetNotFound 252 } 253 254 func (z *SortedSet) ZExist(key string, value []byte) (bool, error) { 255 if sortedSet, ok := z.M[key]; ok { 256 hash, err := getFnv32(value) 257 if err != nil { 258 return false, err 259 } 260 _, ok := sortedSet.dict[hash] 261 return ok, nil 262 } 263 return false, ErrSortedSetNotFound 264 } 265 266 // SCORE represents the score type. 267 type SCORE float64 268 269 // SkipListLevel records forward and span. 270 type SkipListLevel struct { 271 forward *SkipListNode 272 span int64 273 } 274 275 // The SkipList represents the sorted set. 276 type SkipList struct { 277 db *DB 278 header *SkipListNode 279 tail *SkipListNode 280 length int64 281 level int 282 dict map[uint32]*SkipListNode 283 } 284 285 // SkipListNode represents a node in the SkipList. 286 type SkipListNode struct { 287 hash uint32 // unique key of this node 288 record *Record // associated data 289 score SCORE // score to determine the order of this node in the set 290 backward *SkipListNode 291 level []SkipListLevel 292 } 293 294 // Hash returns the key of the node. 295 func (sln *SkipListNode) Hash() uint32 { 296 return sln.hash 297 } 298 299 // Score returns the score of the node. 300 func (sln *SkipListNode) Score() SCORE { 301 return sln.score 302 } 303 304 // createNode returns a newly initialized SkipListNode Object that implements the SkipListNode. 305 func createNode(level int, score SCORE, hash uint32, record *Record) *SkipListNode { 306 node := SkipListNode{ 307 hash: hash, 308 record: record, 309 score: score, 310 level: make([]SkipListLevel, level), 311 } 312 return &node 313 } 314 315 // randomLevel returns a random level for the new skiplist node we are going to create. 316 // The return value of this function is between 1 and SkipListMaxLevel 317 // (both inclusive), with a powerlaw-alike distribution where higher 318 // levels are lesl likely to be returned. 319 func randomLevel() int { 320 level := 1 321 322 for float64(rand.Int31()&0xFFFF) < SkipListP*0xFFFF { 323 level += 1 324 } 325 if level < SkipListMaxLevel { 326 return level 327 } 328 329 return SkipListMaxLevel 330 } 331 332 func newSkipList(db *DB) *SkipList { 333 skipList := &SkipList{ 334 db: db, 335 level: 1, 336 dict: make(map[uint32]*SkipListNode), 337 } 338 hash, _ := getFnv32([]byte("")) 339 skipList.header = createNode(SkipListMaxLevel, 0, hash, nil) 340 return skipList 341 } 342 343 func (sl *SkipList) cmp(r1 *Record, r2 *Record) int { 344 val1, _ := sl.db.getValueByRecord(r1) 345 val2, _ := sl.db.getValueByRecord(r2) 346 return bytes.Compare(val1, val2) 347 } 348 349 func (sl *SkipList) insertNode(score SCORE, hash uint32, record *Record) *SkipListNode { 350 var update [SkipListMaxLevel]*SkipListNode 351 var rank [SkipListMaxLevel]int64 352 353 x := sl.header 354 for i := sl.level - 1; i >= 0; i-- { 355 // store rank that is crosled to reach the insert position 356 if sl.level-1 == i { 357 rank[i] = 0 358 } else { 359 rank[i] = rank[i+1] 360 } 361 362 for x.level[i].forward != nil && 363 (x.level[i].forward.score < score || 364 (x.level[i].forward.score == score && // score is the same but the key is different 365 sl.cmp(x.level[i].forward.record, record) < 0)) { 366 rank[i] += x.level[i].span 367 x = x.level[i].forward 368 } 369 370 update[i] = x 371 } 372 373 /* we assume the key is not already inside, since we allow duplicated 374 * scores, and the re-insertion of score and redis object should never 375 * happen since the caller of Insert() should test in the hash table 376 * if the element is already inside or not. */ 377 level := randomLevel() 378 379 if level > sl.level { // add a new level 380 for i := sl.level; i < level; i++ { 381 rank[i] = 0 382 update[i] = sl.header 383 update[i].level[i].span = sl.length 384 } 385 sl.level = level 386 } 387 388 x = createNode(level, score, hash, record) 389 for i := 0; i < level; i++ { 390 x.level[i].forward = update[i].level[i].forward 391 update[i].level[i].forward = x 392 393 /* update span covered by update[i] as x is inserted here */ 394 x.level[i].span = update[i].level[i].span - (rank[0] - rank[i]) 395 396 update[i].level[i].span = (rank[0] - rank[i]) + 1 397 } 398 399 // increment span for untouched levels 400 for i := level; i < sl.level; i++ { 401 update[i].level[i].span++ 402 } 403 404 if update[0] == sl.header { 405 x.backward = nil 406 } else { 407 x.backward = update[0] 408 } 409 410 if x.level[0].forward != nil { 411 x.level[0].forward.backward = x 412 } else { 413 sl.tail = x 414 } 415 416 sl.length++ 417 418 return x 419 } 420 421 // deleteNode represents internal function used by delete, DeleteByScore and DeleteByRank. 422 func (sl *SkipList) deleteNode(x *SkipListNode, update [SkipListMaxLevel]*SkipListNode) { 423 for i := 0; i < sl.level; i++ { 424 if update[i].level[i].forward == x { 425 update[i].level[i].span += x.level[i].span - 1 426 update[i].level[i].forward = x.level[i].forward 427 } else { 428 update[i].level[i].span -= 1 429 } 430 } 431 if x.level[0].forward != nil { 432 x.level[0].forward.backward = x.backward 433 } else { 434 sl.tail = x.backward 435 } 436 for sl.level > 1 && sl.header.level[sl.level-1].forward == nil { 437 sl.level-- 438 } 439 sl.length-- 440 delete(sl.dict, x.hash) 441 } 442 443 // delete removes an element with matching score/key from the skiplist. 444 func (sl *SkipList) delete(score SCORE, hash uint32) bool { 445 var update [SkipListMaxLevel]*SkipListNode 446 447 targetNode := sl.dict[hash] 448 449 x := sl.header 450 for i := sl.level - 1; i >= 0; i-- { 451 for x.level[i].forward != nil && 452 (x.level[i].forward.score < score || 453 (x.level[i].forward.score == score && 454 sl.cmp(x.level[i].forward.record, targetNode.record) < 0)) { 455 x = x.level[i].forward 456 } 457 update[i] = x 458 } 459 /* We may have multiple elements with the same score, what we need 460 * is to find the element with both the right score and object. */ 461 x = x.level[0].forward 462 if x != nil && score == x.score && sl.cmp(x.record, targetNode.record) == 0 { 463 sl.deleteNode(x, update) 464 // free x 465 return true 466 } 467 return false /* not found */ 468 } 469 470 // Size returns the number of elements in the SkipList. 471 func (sl *SkipList) Size() int { 472 return int(sl.length) 473 } 474 475 // PeekMin returns the element with minimum score, nil if the set is empty. 476 // 477 // Time complexity of this method is : O(log(N)). 478 func (sl *SkipList) PeekMin() *SkipListNode { 479 return sl.header.level[0].forward 480 } 481 482 // PopMin returns and remove the element with minimal score, nil if the set is empty. 483 // 484 // Time complexity of this method is : O(log(N)). 485 func (sl *SkipList) PopMin() *SkipListNode { 486 x := sl.header.level[0].forward 487 if x != nil { 488 sl.Remove(x.hash) 489 } 490 return x 491 } 492 493 // PeekMax returns the element with maximum score, nil if the set is empty. 494 // 495 // Time Complexity : O(1). 496 func (sl *SkipList) PeekMax() *SkipListNode { 497 return sl.tail 498 } 499 500 // PopMax returns and remove the element with maximum score, nil if the set is empty. 501 // 502 // Time complexity of this method is : O(log(N)). 503 func (sl *SkipList) PopMax() *SkipListNode { 504 x := sl.tail 505 if x != nil { 506 sl.Remove(x.hash) 507 } 508 return x 509 } 510 511 // Put puts an element into the sorted set with specific key / value / score. 512 // 513 // Time complexity of this method is : O(log(N)). 514 func (sl *SkipList) Put(score SCORE, value []byte, record *Record) error { 515 var newNode *SkipListNode 516 517 hash, _ := getFnv32(value) 518 519 if n, ok := sl.dict[hash]; ok { 520 // score does not change, only update value 521 if n.score != score { // score changes, delete and re-insert 522 sl.delete(n.score, n.hash) 523 newNode = sl.insertNode(score, hash, record) 524 } 525 } else { 526 newNode = sl.insertNode(score, hash, record) 527 } 528 529 if newNode != nil { 530 sl.dict[hash] = newNode 531 } 532 533 return nil 534 } 535 536 // Remove removes element specified at given key. 537 // 538 // Time complexity of this method is : O(log(N)). 539 func (sl *SkipList) Remove(hash uint32) *SkipListNode { 540 found := sl.dict[hash] 541 if found != nil { 542 sl.delete(found.score, hash) 543 return found 544 } 545 return nil 546 } 547 548 // GetByScoreRangeOptions represents the options of the GetByScoreRange function. 549 type GetByScoreRangeOptions struct { 550 Limit int // limit the max nodes to return 551 ExcludeStart bool // exclude start value, so it search in interval (start, end] or (start, end) 552 ExcludeEnd bool // exclude end value, so it search in interval [start, end) or (start, end) 553 } 554 555 // GetByScoreRange returns the nodes whose score within the specific range. 556 // If options is nil, it searches in interval [start, end] without any limit by default. 557 // 558 // Time complexity of this method is : O(log(N)). 559 func (sl *SkipList) GetByScoreRange(start SCORE, end SCORE, options *GetByScoreRangeOptions) []*SkipListNode { 560 limit := 1<<31 - 1 561 if options != nil && options.Limit > 0 { 562 limit = options.Limit 563 } 564 565 excludeStart := options != nil && options.ExcludeStart 566 excludeEnd := options != nil && options.ExcludeEnd 567 reverse := start > end 568 if reverse { 569 start, end = end, start 570 excludeStart, excludeEnd = excludeEnd, excludeStart 571 } 572 573 var nodes []*SkipListNode 574 575 // determine if out of range 576 if sl.length == 0 { 577 return nodes 578 } 579 580 if reverse { 581 // search from end to start 582 return sl.searchReverse(nodes, excludeStart, excludeEnd, start, end, limit) 583 } 584 // search from start to end 585 return sl.searchForward(nodes, excludeStart, excludeEnd, start, end, limit) 586 } 587 588 func (sl *SkipList) searchForward(nodes []*SkipListNode, excludeStart, excludeEnd bool, start, end SCORE, limit int) []*SkipListNode { 589 // search from start to end 590 x := sl.header 591 if excludeStart { 592 for i := sl.level - 1; i >= 0; i-- { 593 for x.level[i].forward != nil && 594 x.level[i].forward.score <= start { 595 x = x.level[i].forward 596 } 597 } 598 } else { 599 for i := sl.level - 1; i >= 0; i-- { 600 for x.level[i].forward != nil && 601 x.level[i].forward.score < start { 602 x = x.level[i].forward 603 } 604 } 605 } 606 607 /* Current node is the last with score < or <= start. */ 608 x = x.level[0].forward 609 610 for x != nil && limit > 0 { 611 if excludeEnd { 612 if x.score >= end { 613 break 614 } 615 } else { 616 if x.score > end { 617 break 618 } 619 } 620 621 next := x.level[0].forward 622 623 nodes = append(nodes, x) 624 limit-- 625 626 x = next 627 } 628 629 return nodes 630 } 631 632 func (sl *SkipList) searchReverse(nodes []*SkipListNode, excludeStart, excludeEnd bool, start, end SCORE, limit int) []*SkipListNode { 633 x := sl.header 634 635 if excludeEnd { 636 for i := sl.level - 1; i >= 0; i-- { 637 for x.level[i].forward != nil && 638 x.level[i].forward.score < end { 639 x = x.level[i].forward 640 } 641 } 642 } else { 643 for i := sl.level - 1; i >= 0; i-- { 644 for x.level[i].forward != nil && 645 x.level[i].forward.score <= end { 646 x = x.level[i].forward 647 } 648 } 649 } 650 651 for x != nil && limit > 0 { 652 if excludeStart { 653 if x.score <= start { 654 break 655 } 656 } else { 657 if x.score < start { 658 break 659 } 660 } 661 662 next := x.backward 663 664 nodes = append(nodes, x) 665 limit-- 666 667 x = next 668 } 669 670 return nodes 671 } 672 673 // GetByRankRange returns nodes within specific rank range [start, end]. 674 // Note that the rank is 1-based integer. Rank 1 means the first node; Rank -1 means the last node 675 // If start is greater than end, the returned array is in reserved order 676 // If remove is true, the returned nodes are removed. 677 // 678 // Time complexity of this method is : O(log(N)). 679 func (sl *SkipList) GetByRankRange(start, end int, remove bool) []*SkipListNode { 680 var ( 681 update [SkipListMaxLevel]*SkipListNode 682 nodes []*SkipListNode 683 traversed int 684 ) 685 686 start, end = sl.sanitizeIndexes(start, end) 687 688 reverse := start > end 689 if reverse { // swap start and end 690 start, end = end, start 691 } 692 693 traversed = 0 694 x := sl.header 695 for i := sl.level - 1; i >= 0; i-- { 696 for x.level[i].forward != nil && 697 traversed+int(x.level[i].span) < start { 698 traversed += int(x.level[i].span) 699 x = x.level[i].forward 700 } 701 if remove { 702 update[i] = x 703 } else { 704 if traversed+1 == start { 705 break 706 } 707 } 708 } 709 710 traversed++ 711 x = x.level[0].forward 712 for x != nil && traversed <= end { 713 next := x.level[0].forward 714 715 nodes = append(nodes, x) 716 717 if remove { 718 sl.deleteNode(x, update) 719 } 720 721 traversed++ 722 x = next 723 } 724 725 if reverse { 726 for i, j := 0, len(nodes)-1; i < j; i, j = i+1, j-1 { 727 nodes[i], nodes[j] = nodes[j], nodes[i] 728 } 729 } 730 return nodes 731 } 732 733 func (sl *SkipList) sanitizeIndexes(start, end int) (newStart, newEnd int) { 734 if start < 0 { 735 start = int(sl.length) + start + 1 736 } 737 if end < 0 { 738 end = int(sl.length) + end + 1 739 } 740 if start <= 0 { 741 start = 1 742 } 743 if end <= 0 { 744 end = 1 745 } 746 747 return start, end 748 } 749 750 // GetByRank returns the node at given rank. 751 // Note that the rank is 1-based integer. Rank 1 means the first node; Rank -1 means the last node. 752 // If remove is true, the returned nodes are removed 753 // If node is not found at specific rank, nil is returned. 754 // 755 // Time complexity of this method is : O(log(N)). 756 func (sl *SkipList) GetByRank(rank int, remove bool) *SkipListNode { 757 nodes := sl.GetByRankRange(rank, rank, remove) 758 if len(nodes) == 1 { 759 return nodes[0] 760 } 761 return nil 762 } 763 764 // GetByValue returns the node at given key. 765 // If node is not found, nil is returned 766 // 767 // Time complexity : O(1). 768 func (sl *SkipList) GetByValue(value []byte) *SkipListNode { 769 hash, _ := getFnv32(value) 770 return sl.dict[hash] 771 } 772 773 // FindRank Returns the rank of member in the sorted set stored at key, with the scores ordered from low to high. 774 // Note that the rank is 1-based integer. Rank 1 means the first node 775 // If the node is not found, 0 is returned. Otherwise rank(> 0) is returned. 776 // 777 // Time complexity of this method is : O(log(N)). 778 func (sl *SkipList) FindRank(hash uint32) int { 779 rank := 0 780 targetNode := sl.dict[hash] 781 if targetNode != nil { 782 x := sl.header 783 for i := sl.level - 1; i >= 0; i-- { 784 for x.level[i].forward != nil && 785 (x.level[i].forward.score < targetNode.score || 786 (x.level[i].forward.score == targetNode.score && 787 sl.cmp(x.level[i].forward.record, targetNode.record) <= 0)) { 788 rank += int(x.level[i].span) 789 x = x.level[i].forward 790 } 791 792 if x.hash == hash { 793 return rank 794 } 795 } 796 } 797 return 0 798 } 799 800 // FindRevRank Returns the rank of member in the sorted set stored at key, with the scores ordered from high to low. 801 func (sl *SkipList) FindRevRank(hash uint32) int { 802 if sl.length == 0 { 803 return 0 804 } 805 806 if _, ok := sl.dict[hash]; !ok { 807 return 0 808 } 809 810 return sl.Size() - sl.FindRank(hash) + 1 811 }