github.com/nutsdb/nutsdb@v1.0.4/tx_btree.go (about) 1 // Copyright 2019 The nutsdb Author. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package nutsdb 16 17 import ( 18 "errors" 19 "math" 20 "math/big" 21 "strconv" 22 "sync/atomic" 23 "time" 24 25 "github.com/xujiajun/utils/strconv2" 26 ) 27 28 const ( 29 getAllType uint8 = 0 30 getKeysType uint8 = 1 31 getValuesType uint8 = 2 32 ) 33 34 func (tx *Tx) PutWithTimestamp(bucket string, key, value []byte, ttl uint32, timestamp uint64) error { 35 return tx.put(bucket, key, value, ttl, DataSetFlag, timestamp, DataStructureBTree) 36 } 37 38 // Put sets the value for a key in the bucket. 39 // a wrapper of the function put. 40 func (tx *Tx) Put(bucket string, key, value []byte, ttl uint32) error { 41 return tx.put(bucket, key, value, ttl, DataSetFlag, uint64(time.Now().UnixMilli()), DataStructureBTree) 42 } 43 44 // PutIfNotExists set the value for a key in the bucket only if the key doesn't exist already. 45 func (tx *Tx) PutIfNotExists(bucket string, key, value []byte, ttl uint32) error { 46 if err := tx.checkTxIsClosed(); err != nil { 47 return err 48 } 49 50 b, err := tx.db.bm.GetBucket(DataStructureBTree, bucket) 51 if err != nil { 52 return err 53 } 54 bucketId := b.Id 55 56 idx, bucketExists := tx.db.Index.bTree.exist(bucketId) 57 if !bucketExists { 58 return ErrNotFoundBucket 59 } 60 record, recordExists := idx.Find(key) 61 62 if recordExists && !record.IsExpired() { 63 return nil 64 } 65 66 return tx.put(bucket, key, value, ttl, DataSetFlag, uint64(time.Now().UnixMilli()), DataStructureBTree) 67 } 68 69 // PutIfExits set the value for a key in the bucket only if the key already exits. 70 func (tx *Tx) PutIfExists(bucket string, key, value []byte, ttl uint32) error { 71 return tx.update(bucket, key, func(_ []byte) ([]byte, error) { 72 return value, nil 73 }, func(_ uint32) (uint32, error) { 74 return ttl, nil 75 }) 76 } 77 78 // Get retrieves the value for a key in the bucket. 79 // The returned value is only valid for the life of the transaction. 80 func (tx *Tx) Get(bucket string, key []byte) (value []byte, err error) { 81 return tx.get(bucket, key) 82 } 83 84 func (tx *Tx) get(bucket string, key []byte) (value []byte, err error) { 85 if err := tx.checkTxIsClosed(); err != nil { 86 return nil, err 87 } 88 89 b, err := tx.db.bm.GetBucket(DataStructureBTree, bucket) 90 if err != nil { 91 return nil, err 92 } 93 bucketId := b.Id 94 95 bucketStatus := tx.getBucketStatus(DataStructureBTree, bucket) 96 if bucketStatus == BucketStatusDeleted { 97 return nil, ErrBucketNotFound 98 } 99 100 status, entry := tx.findEntryAndItsStatus(DataStructureBTree, bucket, string(key)) 101 if status != NotFoundEntry && entry != nil { 102 if status == EntryDeleted { 103 return nil, ErrKeyNotFound 104 } else { 105 return entry.Value, nil 106 } 107 } 108 109 if idx, ok := tx.db.Index.bTree.exist(bucketId); ok { 110 record, found := idx.Find(key) 111 if !found { 112 return nil, ErrKeyNotFound 113 } 114 115 if record.IsExpired() { 116 tx.putDeleteLog(bucketId, key, nil, Persistent, DataDeleteFlag, uint64(time.Now().Unix()), DataStructureBTree) 117 return nil, ErrNotFoundKey 118 } 119 120 value, err = tx.db.getValueByRecord(record) 121 if err != nil { 122 return nil, err 123 } 124 return value, nil 125 126 } else { 127 return nil, ErrNotFoundBucket 128 } 129 } 130 131 func (tx *Tx) ValueLen(bucket string, key []byte) (int, error) { 132 value, err := tx.get(bucket, key) 133 return len(value), err 134 } 135 136 func (tx *Tx) GetMaxKey(bucket string) ([]byte, error) { 137 return tx.getMaxOrMinKey(bucket, true) 138 } 139 140 func (tx *Tx) GetMinKey(bucket string) ([]byte, error) { 141 return tx.getMaxOrMinKey(bucket, false) 142 } 143 144 func (tx *Tx) getMaxOrMinKey(bucket string, isMax bool) ([]byte, error) { 145 if err := tx.checkTxIsClosed(); err != nil { 146 return nil, err 147 } 148 149 b, err := tx.db.bm.GetBucket(DataStructureBTree, bucket) 150 if err != nil { 151 return nil, err 152 } 153 bucketId := b.Id 154 155 if idx, ok := tx.db.Index.bTree.exist(bucketId); ok { 156 var ( 157 item *Item 158 found bool 159 ) 160 161 if isMax { 162 item, found = idx.Max() 163 } else { 164 item, found = idx.Min() 165 } 166 167 if !found { 168 return nil, ErrKeyNotFound 169 } 170 171 if item.record.IsExpired() { 172 tx.putDeleteLog(bucketId, item.key, nil, Persistent, DataDeleteFlag, uint64(time.Now().Unix()), DataStructureBTree) 173 return nil, ErrNotFoundKey 174 } 175 176 return item.key, nil 177 } else { 178 return nil, ErrNotFoundBucket 179 } 180 } 181 182 // GetAll returns all keys and values of the bucket stored at given bucket. 183 func (tx *Tx) GetAll(bucket string) ([][]byte, [][]byte, error) { 184 return tx.getAllOrKeysOrValues(bucket, getAllType) 185 } 186 187 // GetKeys returns all keys of the bucket stored at given bucket. 188 func (tx *Tx) GetKeys(bucket string) ([][]byte, error) { 189 keys, _, err := tx.getAllOrKeysOrValues(bucket, getKeysType) 190 return keys, err 191 } 192 193 // GetValues returns all values of the bucket stored at given bucket. 194 func (tx *Tx) GetValues(bucket string) ([][]byte, error) { 195 _, values, err := tx.getAllOrKeysOrValues(bucket, getValuesType) 196 return values, err 197 } 198 199 func (tx *Tx) getAllOrKeysOrValues(bucket string, typ uint8) ([][]byte, [][]byte, error) { 200 if err := tx.checkTxIsClosed(); err != nil { 201 return nil, nil, err 202 } 203 204 bucketId, err := tx.db.bm.GetBucketID(DataStructureBTree, bucket) 205 if err != nil { 206 return nil, nil, err 207 } 208 209 if index, ok := tx.db.Index.bTree.exist(bucketId); ok { 210 records := index.All() 211 212 var ( 213 keys [][]byte 214 values [][]byte 215 ) 216 217 switch typ { 218 case getAllType: 219 keys, values, err = tx.getHintIdxDataItemsWrapper(records, ScanNoLimit, bucketId, true, true) 220 case getKeysType: 221 keys, _, err = tx.getHintIdxDataItemsWrapper(records, ScanNoLimit, bucketId, true, false) 222 case getValuesType: 223 _, values, err = tx.getHintIdxDataItemsWrapper(records, ScanNoLimit, bucketId, false, true) 224 } 225 226 if err != nil { 227 return nil, nil, err 228 } 229 230 return keys, values, nil 231 } 232 233 return nil, nil, nil 234 } 235 236 func (tx *Tx) GetSet(bucket string, key, value []byte) (oldValue []byte, err error) { 237 return oldValue, tx.update(bucket, key, func(b []byte) ([]byte, error) { 238 oldValue = b 239 return value, nil 240 }, func(oldTTL uint32) (uint32, error) { 241 return oldTTL, nil 242 }) 243 } 244 245 // RangeScan query a range at given bucket, start and end slice. 246 func (tx *Tx) RangeScan(bucket string, start, end []byte) (values [][]byte, err error) { 247 if err := tx.checkTxIsClosed(); err != nil { 248 return nil, err 249 } 250 b, err := tx.db.bm.GetBucket(DataStructureBTree, bucket) 251 if err != nil { 252 return nil, err 253 } 254 bucketId := b.Id 255 256 if index, ok := tx.db.Index.bTree.exist(bucketId); ok { 257 records := index.Range(start, end) 258 if err != nil { 259 return nil, ErrRangeScan 260 } 261 262 _, values, err = tx.getHintIdxDataItemsWrapper(records, ScanNoLimit, bucketId, false, true) 263 if err != nil { 264 return nil, ErrRangeScan 265 } 266 } 267 268 if len(values) == 0 { 269 return nil, ErrRangeScan 270 } 271 272 return 273 } 274 275 // PrefixScan iterates over a key prefix at given bucket, prefix and limitNum. 276 // LimitNum will limit the number of entries return. 277 func (tx *Tx) PrefixScan(bucket string, prefix []byte, offsetNum int, limitNum int) (values [][]byte, err error) { 278 if err := tx.checkTxIsClosed(); err != nil { 279 return nil, err 280 } 281 b, err := tx.db.bm.GetBucket(DataStructureBTree, bucket) 282 if err != nil { 283 return nil, err 284 } 285 bucketId := b.Id 286 287 if idx, ok := tx.db.Index.bTree.exist(bucketId); ok { 288 records := idx.PrefixScan(prefix, offsetNum, limitNum) 289 _, values, err = tx.getHintIdxDataItemsWrapper(records, limitNum, bucketId, false, true) 290 if err != nil { 291 return nil, ErrPrefixScan 292 } 293 } 294 295 if len(values) == 0 { 296 return nil, ErrPrefixScan 297 } 298 299 return 300 } 301 302 // PrefixSearchScan iterates over a key prefix at given bucket, prefix, match regular expression and limitNum. 303 // LimitNum will limit the number of entries return. 304 func (tx *Tx) PrefixSearchScan(bucket string, prefix []byte, reg string, offsetNum int, limitNum int) (values [][]byte, err error) { 305 if err := tx.checkTxIsClosed(); err != nil { 306 return nil, err 307 } 308 b, err := tx.db.bm.GetBucket(DataStructureBTree, bucket) 309 if err != nil { 310 return nil, err 311 } 312 bucketId := b.Id 313 314 if idx, ok := tx.db.Index.bTree.exist(bucketId); ok { 315 records := idx.PrefixSearchScan(prefix, reg, offsetNum, limitNum) 316 _, values, err = tx.getHintIdxDataItemsWrapper(records, limitNum, bucketId, false, true) 317 if err != nil { 318 return nil, ErrPrefixSearchScan 319 } 320 } 321 322 if len(values) == 0 { 323 return nil, ErrPrefixSearchScan 324 } 325 326 return 327 } 328 329 // Delete removes a key from the bucket at given bucket and key. 330 func (tx *Tx) Delete(bucket string, key []byte) error { 331 if err := tx.checkTxIsClosed(); err != nil { 332 return err 333 } 334 b, err := tx.db.bm.GetBucket(DataStructureBTree, bucket) 335 if err != nil { 336 return err 337 } 338 bucketId := b.Id 339 340 if idx, ok := tx.db.Index.bTree.exist(bucketId); ok { 341 if _, found := idx.Find(key); !found { 342 return ErrKeyNotFound 343 } 344 } else { 345 return ErrNotFoundBucket 346 } 347 348 return tx.put(bucket, key, nil, Persistent, DataDeleteFlag, uint64(time.Now().Unix()), DataStructureBTree) 349 } 350 351 // getHintIdxDataItemsWrapper returns keys and values when prefix scanning or range scanning. 352 func (tx *Tx) getHintIdxDataItemsWrapper(records []*Record, limitNum int, bucketId BucketId, needKeys bool, needValues bool) (keys [][]byte, values [][]byte, err error) { 353 for _, record := range records { 354 if record.IsExpired() { 355 tx.putDeleteLog(bucketId, record.Key, nil, Persistent, DataDeleteFlag, uint64(time.Now().Unix()), DataStructureBTree) 356 continue 357 } 358 359 if limitNum > 0 && len(values) < limitNum || limitNum == ScanNoLimit { 360 if needKeys { 361 keys = append(keys, record.Key) 362 } 363 364 if needValues { 365 value, err := tx.db.getValueByRecord(record) 366 if err != nil { 367 return nil, nil, err 368 } 369 values = append(values, value) 370 } 371 } 372 } 373 374 return keys, values, nil 375 } 376 377 func (tx *Tx) tryGet(bucket string, key []byte, solveRecord func(record *Record, found bool, bucketId BucketId) error) error { 378 if err := tx.checkTxIsClosed(); err != nil { 379 return err 380 } 381 382 bucketId, err := tx.db.bm.GetBucketID(DataStructureBTree, bucket) 383 if err != nil { 384 return err 385 } 386 387 if idx, ok := tx.db.Index.bTree.exist(bucketId); ok { 388 record, found := idx.Find(key) 389 return solveRecord(record, found, bucketId) 390 } else { 391 return ErrBucketNotFound 392 } 393 } 394 395 func (tx *Tx) update(bucket string, key []byte, getNewValue func([]byte) ([]byte, error), getNewTTL func(uint32) (uint32, error)) error { 396 return tx.tryGet(bucket, key, func(record *Record, found bool, bucketId BucketId) error { 397 if !found { 398 return ErrKeyNotFound 399 } 400 401 if record.IsExpired() { 402 tx.putDeleteLog(bucketId, key, nil, Persistent, DataDeleteFlag, uint64(time.Now().Unix()), DataStructureBTree) 403 return ErrNotFoundKey 404 } 405 406 value, err := tx.db.getValueByRecord(record) 407 if err != nil { 408 return err 409 } 410 newValue, err := getNewValue(value) 411 if err != nil { 412 return err 413 } 414 415 newTTL, err := getNewTTL(record.TTL) 416 if err != nil { 417 return err 418 } 419 420 return tx.put(bucket, key, newValue, newTTL, DataSetFlag, uint64(time.Now().Unix()), DataStructureBTree) 421 }) 422 } 423 424 func (tx *Tx) updateOrPut(bucket string, key, value []byte, getUpdatedValue func([]byte) ([]byte, error)) error { 425 return tx.tryGet(bucket, key, func(record *Record, found bool, bucketId BucketId) error { 426 if !found { 427 return tx.put(bucket, key, value, Persistent, DataSetFlag, uint64(time.Now().Unix()), DataStructureBTree) 428 } 429 430 value, err := tx.db.getValueByRecord(record) 431 if err != nil { 432 return err 433 } 434 newValue, err := getUpdatedValue(value) 435 if err != nil { 436 return err 437 } 438 439 return tx.put(bucket, key, newValue, record.TTL, DataSetFlag, uint64(time.Now().Unix()), DataStructureBTree) 440 }) 441 } 442 443 func bigIntIncr(a string, b string) string { 444 bigIntA, _ := new(big.Int).SetString(a, 10) 445 bigIntB, _ := new(big.Int).SetString(b, 10) 446 bigIntA.Add(bigIntA, bigIntB) 447 return bigIntA.String() 448 } 449 450 func (tx *Tx) integerIncr(bucket string, key []byte, increment int64) error { 451 return tx.update(bucket, key, func(value []byte) ([]byte, error) { 452 intValue, err := strconv2.StrToInt64(string(value)) 453 454 if err != nil && errors.Is(err, strconv.ErrRange) { 455 return []byte(bigIntIncr(string(value), strconv2.Int64ToStr(increment))), nil 456 } 457 458 if err != nil { 459 return nil, ErrValueNotInteger 460 } 461 462 if (increment > 0 && math.MaxInt64-increment < intValue) || (increment < 0 && math.MinInt64-increment > intValue) { 463 return []byte(bigIntIncr(string(value), strconv2.Int64ToStr(increment))), nil 464 } 465 466 atomic.AddInt64(&intValue, increment) 467 return []byte(strconv2.Int64ToStr(intValue)), nil 468 }, func(oldTTL uint32) (uint32, error) { 469 return oldTTL, nil 470 }) 471 } 472 473 func (tx *Tx) Incr(bucket string, key []byte) error { 474 return tx.integerIncr(bucket, key, 1) 475 } 476 477 func (tx *Tx) Decr(bucket string, key []byte) error { 478 return tx.integerIncr(bucket, key, -1) 479 } 480 481 func (tx *Tx) IncrBy(bucket string, key []byte, increment int64) error { 482 return tx.integerIncr(bucket, key, increment) 483 } 484 485 func (tx *Tx) DecrBy(bucket string, key []byte, decrement int64) error { 486 return tx.integerIncr(bucket, key, -1*decrement) 487 } 488 489 func (tx *Tx) GetBit(bucket string, key []byte, offset int) (byte, error) { 490 if offset >= math.MaxInt || offset < 0 { 491 return 0, ErrOffsetInvalid 492 } 493 494 value, err := tx.Get(bucket, key) 495 if err != nil { 496 return 0, err 497 } 498 499 if len(value) <= offset { 500 return 0, nil 501 } 502 503 return value[offset], nil 504 } 505 506 func (tx *Tx) SetBit(bucket string, key []byte, offset int, bit byte) error { 507 if offset >= math.MaxInt || offset < 0 { 508 return ErrOffsetInvalid 509 } 510 511 valueIfKeyNotFound := make([]byte, offset+1) 512 valueIfKeyNotFound[offset] = bit 513 514 return tx.updateOrPut(bucket, key, valueIfKeyNotFound, func(value []byte) ([]byte, error) { 515 if len(value) <= offset { 516 value = append(value, make([]byte, offset-len(value)+1)...) 517 value[offset] = bit 518 return value, nil 519 } else { 520 value[offset] = bit 521 return value, nil 522 } 523 }) 524 } 525 526 // GetTTL returns remaining TTL of a value by key. 527 // It returns 528 // (-1, nil) If TTL is Persistent 529 // (0, ErrBucketNotFound|ErrKeyNotFound) If expired or not found 530 // (TTL, nil) If the record exists with a TTL 531 // Note: The returned remaining TTL will be in seconds. For example, 532 // remainingTTL is 500ms, It'll return 0. 533 func (tx *Tx) GetTTL(bucket string, key []byte) (int64, error) { 534 if err := tx.checkTxIsClosed(); err != nil { 535 return 0, err 536 } 537 538 bucketId, err := tx.db.bm.GetBucketID(DataStructureBTree, bucket) 539 if err != nil { 540 return 0, err 541 } 542 idx, bucketExists := tx.db.Index.bTree.exist(bucketId) 543 544 if !bucketExists { 545 return 0, ErrBucketNotFound 546 } 547 548 record, recordFound := idx.Find(key) 549 550 if !recordFound || record.IsExpired() { 551 return 0, ErrKeyNotFound 552 } 553 554 if record.TTL == Persistent { 555 return -1, nil 556 } 557 558 remTTL := tx.db.expireTime(record.Timestamp, record.TTL) 559 if remTTL >= 0 { 560 return int64(remTTL.Seconds()), nil 561 } else { 562 return 0, ErrKeyNotFound 563 } 564 } 565 566 // Persist updates record's TTL as Persistent if the record exits. 567 func (tx *Tx) Persist(bucket string, key []byte) error { 568 return tx.update(bucket, key, func(oldValue []byte) ([]byte, error) { 569 return oldValue, nil 570 }, func(_ uint32) (uint32, error) { 571 return Persistent, nil 572 }) 573 } 574 575 func (tx *Tx) MSet(bucket string, ttl uint32, args ...[]byte) error { 576 if len(args) == 0 { 577 return nil 578 } 579 580 if len(args)%2 != 0 { 581 return ErrKVArgsLenNotEven 582 } 583 584 for i := 0; i < len(args); i += 2 { 585 if err := tx.put(bucket, args[i], args[i+1], ttl, DataSetFlag, uint64(time.Now().Unix()), DataStructureBTree); err != nil { 586 return err 587 } 588 } 589 590 return nil 591 } 592 593 func (tx *Tx) MGet(bucket string, keys ...[]byte) ([][]byte, error) { 594 if len(keys) == 0 { 595 return nil, nil 596 } 597 598 values := make([][]byte, len(keys)) 599 for i, key := range keys { 600 value, err := tx.Get(bucket, key) 601 if err != nil { 602 return nil, err 603 } 604 values[i] = value 605 } 606 607 return values, nil 608 } 609 610 func (tx *Tx) Append(bucket string, key, appendage []byte) error { 611 if len(appendage) == 0 { 612 return nil 613 } 614 615 return tx.updateOrPut(bucket, key, appendage, func(value []byte) ([]byte, error) { 616 return append(value, appendage...), nil 617 }) 618 } 619 620 func (tx *Tx) GetRange(bucket string, key []byte, start, end int) ([]byte, error) { 621 if start > end { 622 return nil, ErrStartGreaterThanEnd 623 } 624 625 value, err := tx.get(bucket, key) 626 if err != nil { 627 return nil, err 628 } 629 630 if start >= len(value) { 631 return nil, nil 632 } 633 634 if end >= len(value) { 635 return value[start:], nil 636 } 637 638 return value[start : end+1], nil 639 }