github.com/insionng/yougam@v0.0.0-20170714101924-2bc18d833463/libraries/syndtr/goleveldb/leveldb/table/reader.go (about) 1 // Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com> 2 // All rights reserved. 3 // 4 // Use of this source code is governed by a BSD-style license that can be 5 // found in the LICENSE file. 6 7 package table 8 9 import ( 10 "encoding/binary" 11 "fmt" 12 "io" 13 "sort" 14 "strings" 15 "sync" 16 17 "github.com/insionng/yougam/libraries/golang/snappy" 18 19 "github.com/insionng/yougam/libraries/syndtr/goleveldb/leveldb/cache" 20 "github.com/insionng/yougam/libraries/syndtr/goleveldb/leveldb/comparer" 21 "github.com/insionng/yougam/libraries/syndtr/goleveldb/leveldb/errors" 22 "github.com/insionng/yougam/libraries/syndtr/goleveldb/leveldb/filter" 23 "github.com/insionng/yougam/libraries/syndtr/goleveldb/leveldb/iterator" 24 "github.com/insionng/yougam/libraries/syndtr/goleveldb/leveldb/opt" 25 "github.com/insionng/yougam/libraries/syndtr/goleveldb/leveldb/storage" 26 "github.com/insionng/yougam/libraries/syndtr/goleveldb/leveldb/util" 27 ) 28 29 var ( 30 ErrNotFound = errors.ErrNotFound 31 ErrReaderReleased = errors.New("leveldb/table: reader released") 32 ErrIterReleased = errors.New("leveldb/table: iterator released") 33 ) 34 35 type ErrCorrupted struct { 36 Pos int64 37 Size int64 38 Kind string 39 Reason string 40 } 41 42 func (e *ErrCorrupted) Error() string { 43 return fmt.Sprintf("leveldb/table: corruption on %s (pos=%d): %s", e.Kind, e.Pos, e.Reason) 44 } 45 46 func max(x, y int) int { 47 if x > y { 48 return x 49 } 50 return y 51 } 52 53 type block struct { 54 bpool *util.BufferPool 55 bh blockHandle 56 data []byte 57 restartsLen int 58 restartsOffset int 59 } 60 61 func (b *block) seek(cmp comparer.Comparer, rstart, rlimit int, key []byte) (index, offset int, err error) { 62 index = sort.Search(b.restartsLen-rstart-(b.restartsLen-rlimit), func(i int) bool { 63 offset := int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*(rstart+i):])) 64 offset += 1 // shared always zero, since this is a restart point 65 v1, n1 := binary.Uvarint(b.data[offset:]) // key length 66 _, n2 := binary.Uvarint(b.data[offset+n1:]) // value length 67 m := offset + n1 + n2 68 return cmp.Compare(b.data[m:m+int(v1)], key) > 0 69 }) + rstart - 1 70 if index < rstart { 71 // The smallest key is greater-than key sought. 72 index = rstart 73 } 74 offset = int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*index:])) 75 return 76 } 77 78 func (b *block) restartIndex(rstart, rlimit, offset int) int { 79 return sort.Search(b.restartsLen-rstart-(b.restartsLen-rlimit), func(i int) bool { 80 return int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*(rstart+i):])) > offset 81 }) + rstart - 1 82 } 83 84 func (b *block) restartOffset(index int) int { 85 return int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*index:])) 86 } 87 88 func (b *block) entry(offset int) (key, value []byte, nShared, n int, err error) { 89 if offset >= b.restartsOffset { 90 if offset != b.restartsOffset { 91 err = &ErrCorrupted{Reason: "entries offset not aligned"} 92 } 93 return 94 } 95 v0, n0 := binary.Uvarint(b.data[offset:]) // Shared prefix length 96 v1, n1 := binary.Uvarint(b.data[offset+n0:]) // Key length 97 v2, n2 := binary.Uvarint(b.data[offset+n0+n1:]) // Value length 98 m := n0 + n1 + n2 99 n = m + int(v1) + int(v2) 100 if n0 <= 0 || n1 <= 0 || n2 <= 0 || offset+n > b.restartsOffset { 101 err = &ErrCorrupted{Reason: "entries corrupted"} 102 return 103 } 104 key = b.data[offset+m : offset+m+int(v1)] 105 value = b.data[offset+m+int(v1) : offset+n] 106 nShared = int(v0) 107 return 108 } 109 110 func (b *block) Release() { 111 b.bpool.Put(b.data) 112 b.bpool = nil 113 b.data = nil 114 } 115 116 type dir int 117 118 const ( 119 dirReleased dir = iota - 1 120 dirSOI 121 dirEOI 122 dirBackward 123 dirForward 124 ) 125 126 type blockIter struct { 127 tr *Reader 128 block *block 129 blockReleaser util.Releaser 130 releaser util.Releaser 131 key, value []byte 132 offset int 133 // Previous offset, only filled by Next. 134 prevOffset int 135 prevNode []int 136 prevKeys []byte 137 restartIndex int 138 // Iterator direction. 139 dir dir 140 // Restart index slice range. 141 riStart int 142 riLimit int 143 // Offset slice range. 144 offsetStart int 145 offsetRealStart int 146 offsetLimit int 147 // Error. 148 err error 149 } 150 151 func (i *blockIter) sErr(err error) { 152 i.err = err 153 i.key = nil 154 i.value = nil 155 i.prevNode = nil 156 i.prevKeys = nil 157 } 158 159 func (i *blockIter) reset() { 160 if i.dir == dirBackward { 161 i.prevNode = i.prevNode[:0] 162 i.prevKeys = i.prevKeys[:0] 163 } 164 i.restartIndex = i.riStart 165 i.offset = i.offsetStart 166 i.dir = dirSOI 167 i.key = i.key[:0] 168 i.value = nil 169 } 170 171 func (i *blockIter) isFirst() bool { 172 switch i.dir { 173 case dirForward: 174 return i.prevOffset == i.offsetRealStart 175 case dirBackward: 176 return len(i.prevNode) == 1 && i.restartIndex == i.riStart 177 } 178 return false 179 } 180 181 func (i *blockIter) isLast() bool { 182 switch i.dir { 183 case dirForward, dirBackward: 184 return i.offset == i.offsetLimit 185 } 186 return false 187 } 188 189 func (i *blockIter) First() bool { 190 if i.err != nil { 191 return false 192 } else if i.dir == dirReleased { 193 i.err = ErrIterReleased 194 return false 195 } 196 197 if i.dir == dirBackward { 198 i.prevNode = i.prevNode[:0] 199 i.prevKeys = i.prevKeys[:0] 200 } 201 i.dir = dirSOI 202 return i.Next() 203 } 204 205 func (i *blockIter) Last() bool { 206 if i.err != nil { 207 return false 208 } else if i.dir == dirReleased { 209 i.err = ErrIterReleased 210 return false 211 } 212 213 if i.dir == dirBackward { 214 i.prevNode = i.prevNode[:0] 215 i.prevKeys = i.prevKeys[:0] 216 } 217 i.dir = dirEOI 218 return i.Prev() 219 } 220 221 func (i *blockIter) Seek(key []byte) bool { 222 if i.err != nil { 223 return false 224 } else if i.dir == dirReleased { 225 i.err = ErrIterReleased 226 return false 227 } 228 229 ri, offset, err := i.block.seek(i.tr.cmp, i.riStart, i.riLimit, key) 230 if err != nil { 231 i.sErr(err) 232 return false 233 } 234 i.restartIndex = ri 235 i.offset = max(i.offsetStart, offset) 236 if i.dir == dirSOI || i.dir == dirEOI { 237 i.dir = dirForward 238 } 239 for i.Next() { 240 if i.tr.cmp.Compare(i.key, key) >= 0 { 241 return true 242 } 243 } 244 return false 245 } 246 247 func (i *blockIter) Next() bool { 248 if i.dir == dirEOI || i.err != nil { 249 return false 250 } else if i.dir == dirReleased { 251 i.err = ErrIterReleased 252 return false 253 } 254 255 if i.dir == dirSOI { 256 i.restartIndex = i.riStart 257 i.offset = i.offsetStart 258 } else if i.dir == dirBackward { 259 i.prevNode = i.prevNode[:0] 260 i.prevKeys = i.prevKeys[:0] 261 } 262 for i.offset < i.offsetRealStart { 263 key, value, nShared, n, err := i.block.entry(i.offset) 264 if err != nil { 265 i.sErr(i.tr.fixErrCorruptedBH(i.block.bh, err)) 266 return false 267 } 268 if n == 0 { 269 i.dir = dirEOI 270 return false 271 } 272 i.key = append(i.key[:nShared], key...) 273 i.value = value 274 i.offset += n 275 } 276 if i.offset >= i.offsetLimit { 277 i.dir = dirEOI 278 if i.offset != i.offsetLimit { 279 i.sErr(i.tr.newErrCorruptedBH(i.block.bh, "entries offset not aligned")) 280 } 281 return false 282 } 283 key, value, nShared, n, err := i.block.entry(i.offset) 284 if err != nil { 285 i.sErr(i.tr.fixErrCorruptedBH(i.block.bh, err)) 286 return false 287 } 288 if n == 0 { 289 i.dir = dirEOI 290 return false 291 } 292 i.key = append(i.key[:nShared], key...) 293 i.value = value 294 i.prevOffset = i.offset 295 i.offset += n 296 i.dir = dirForward 297 return true 298 } 299 300 func (i *blockIter) Prev() bool { 301 if i.dir == dirSOI || i.err != nil { 302 return false 303 } else if i.dir == dirReleased { 304 i.err = ErrIterReleased 305 return false 306 } 307 308 var ri int 309 if i.dir == dirForward { 310 // Change direction. 311 i.offset = i.prevOffset 312 if i.offset == i.offsetRealStart { 313 i.dir = dirSOI 314 return false 315 } 316 ri = i.block.restartIndex(i.restartIndex, i.riLimit, i.offset) 317 i.dir = dirBackward 318 } else if i.dir == dirEOI { 319 // At the end of iterator. 320 i.restartIndex = i.riLimit 321 i.offset = i.offsetLimit 322 if i.offset == i.offsetRealStart { 323 i.dir = dirSOI 324 return false 325 } 326 ri = i.riLimit - 1 327 i.dir = dirBackward 328 } else if len(i.prevNode) == 1 { 329 // This is the end of a restart range. 330 i.offset = i.prevNode[0] 331 i.prevNode = i.prevNode[:0] 332 if i.restartIndex == i.riStart { 333 i.dir = dirSOI 334 return false 335 } 336 i.restartIndex-- 337 ri = i.restartIndex 338 } else { 339 // In the middle of restart range, get from cache. 340 n := len(i.prevNode) - 3 341 node := i.prevNode[n:] 342 i.prevNode = i.prevNode[:n] 343 // Get the key. 344 ko := node[0] 345 i.key = append(i.key[:0], i.prevKeys[ko:]...) 346 i.prevKeys = i.prevKeys[:ko] 347 // Get the value. 348 vo := node[1] 349 vl := vo + node[2] 350 i.value = i.block.data[vo:vl] 351 i.offset = vl 352 return true 353 } 354 // Build entries cache. 355 i.key = i.key[:0] 356 i.value = nil 357 offset := i.block.restartOffset(ri) 358 if offset == i.offset { 359 ri -= 1 360 if ri < 0 { 361 i.dir = dirSOI 362 return false 363 } 364 offset = i.block.restartOffset(ri) 365 } 366 i.prevNode = append(i.prevNode, offset) 367 for { 368 key, value, nShared, n, err := i.block.entry(offset) 369 if err != nil { 370 i.sErr(i.tr.fixErrCorruptedBH(i.block.bh, err)) 371 return false 372 } 373 if offset >= i.offsetRealStart { 374 if i.value != nil { 375 // Appends 3 variables: 376 // 1. Previous keys offset 377 // 2. Value offset in the data block 378 // 3. Value length 379 i.prevNode = append(i.prevNode, len(i.prevKeys), offset-len(i.value), len(i.value)) 380 i.prevKeys = append(i.prevKeys, i.key...) 381 } 382 i.value = value 383 } 384 i.key = append(i.key[:nShared], key...) 385 offset += n 386 // Stop if target offset reached. 387 if offset >= i.offset { 388 if offset != i.offset { 389 i.sErr(i.tr.newErrCorruptedBH(i.block.bh, "entries offset not aligned")) 390 return false 391 } 392 393 break 394 } 395 } 396 i.restartIndex = ri 397 i.offset = offset 398 return true 399 } 400 401 func (i *blockIter) Key() []byte { 402 if i.err != nil || i.dir <= dirEOI { 403 return nil 404 } 405 return i.key 406 } 407 408 func (i *blockIter) Value() []byte { 409 if i.err != nil || i.dir <= dirEOI { 410 return nil 411 } 412 return i.value 413 } 414 415 func (i *blockIter) Release() { 416 if i.dir != dirReleased { 417 i.tr = nil 418 i.block = nil 419 i.prevNode = nil 420 i.prevKeys = nil 421 i.key = nil 422 i.value = nil 423 i.dir = dirReleased 424 if i.blockReleaser != nil { 425 i.blockReleaser.Release() 426 i.blockReleaser = nil 427 } 428 if i.releaser != nil { 429 i.releaser.Release() 430 i.releaser = nil 431 } 432 } 433 } 434 435 func (i *blockIter) SetReleaser(releaser util.Releaser) { 436 if i.dir == dirReleased { 437 panic(util.ErrReleased) 438 } 439 if i.releaser != nil && releaser != nil { 440 panic(util.ErrHasReleaser) 441 } 442 i.releaser = releaser 443 } 444 445 func (i *blockIter) Valid() bool { 446 return i.err == nil && (i.dir == dirBackward || i.dir == dirForward) 447 } 448 449 func (i *blockIter) Error() error { 450 return i.err 451 } 452 453 type filterBlock struct { 454 bpool *util.BufferPool 455 data []byte 456 oOffset int 457 baseLg uint 458 filtersNum int 459 } 460 461 func (b *filterBlock) contains(filter filter.Filter, offset uint64, key []byte) bool { 462 i := int(offset >> b.baseLg) 463 if i < b.filtersNum { 464 o := b.data[b.oOffset+i*4:] 465 n := int(binary.LittleEndian.Uint32(o)) 466 m := int(binary.LittleEndian.Uint32(o[4:])) 467 if n < m && m <= b.oOffset { 468 return filter.Contains(b.data[n:m], key) 469 } else if n == m { 470 return false 471 } 472 } 473 return true 474 } 475 476 func (b *filterBlock) Release() { 477 b.bpool.Put(b.data) 478 b.bpool = nil 479 b.data = nil 480 } 481 482 type indexIter struct { 483 *blockIter 484 tr *Reader 485 slice *util.Range 486 // Options 487 fillCache bool 488 } 489 490 func (i *indexIter) Get() iterator.Iterator { 491 value := i.Value() 492 if value == nil { 493 return nil 494 } 495 dataBH, n := decodeBlockHandle(value) 496 if n == 0 { 497 return iterator.NewEmptyIterator(i.tr.newErrCorruptedBH(i.tr.indexBH, "bad data block handle")) 498 } 499 500 var slice *util.Range 501 if i.slice != nil && (i.blockIter.isFirst() || i.blockIter.isLast()) { 502 slice = i.slice 503 } 504 return i.tr.getDataIterErr(dataBH, slice, i.tr.verifyChecksum, i.fillCache) 505 } 506 507 // Reader is a table reader. 508 type Reader struct { 509 mu sync.RWMutex 510 fd storage.FileDesc 511 reader io.ReaderAt 512 cache *cache.NamespaceGetter 513 err error 514 bpool *util.BufferPool 515 // Options 516 o *opt.Options 517 cmp comparer.Comparer 518 filter filter.Filter 519 verifyChecksum bool 520 521 dataEnd int64 522 metaBH, indexBH, filterBH blockHandle 523 indexBlock *block 524 filterBlock *filterBlock 525 } 526 527 func (r *Reader) blockKind(bh blockHandle) string { 528 switch bh.offset { 529 case r.metaBH.offset: 530 return "meta-block" 531 case r.indexBH.offset: 532 return "index-block" 533 case r.filterBH.offset: 534 if r.filterBH.length > 0 { 535 return "filter-block" 536 } 537 } 538 return "data-block" 539 } 540 541 func (r *Reader) newErrCorrupted(pos, size int64, kind, reason string) error { 542 return &errors.ErrCorrupted{Fd: r.fd, Err: &ErrCorrupted{Pos: pos, Size: size, Kind: kind, Reason: reason}} 543 } 544 545 func (r *Reader) newErrCorruptedBH(bh blockHandle, reason string) error { 546 return r.newErrCorrupted(int64(bh.offset), int64(bh.length), r.blockKind(bh), reason) 547 } 548 549 func (r *Reader) fixErrCorruptedBH(bh blockHandle, err error) error { 550 if cerr, ok := err.(*ErrCorrupted); ok { 551 cerr.Pos = int64(bh.offset) 552 cerr.Size = int64(bh.length) 553 cerr.Kind = r.blockKind(bh) 554 return &errors.ErrCorrupted{Fd: r.fd, Err: cerr} 555 } 556 return err 557 } 558 559 func (r *Reader) readRawBlock(bh blockHandle, verifyChecksum bool) ([]byte, error) { 560 data := r.bpool.Get(int(bh.length + blockTrailerLen)) 561 if _, err := r.reader.ReadAt(data, int64(bh.offset)); err != nil && err != io.EOF { 562 return nil, err 563 } 564 565 if verifyChecksum { 566 n := bh.length + 1 567 checksum0 := binary.LittleEndian.Uint32(data[n:]) 568 checksum1 := util.NewCRC(data[:n]).Value() 569 if checksum0 != checksum1 { 570 r.bpool.Put(data) 571 return nil, r.newErrCorruptedBH(bh, fmt.Sprintf("checksum mismatch, want=%#x got=%#x", checksum0, checksum1)) 572 } 573 } 574 575 switch data[bh.length] { 576 case blockTypeNoCompression: 577 data = data[:bh.length] 578 case blockTypeSnappyCompression: 579 decLen, err := snappy.DecodedLen(data[:bh.length]) 580 if err != nil { 581 return nil, r.newErrCorruptedBH(bh, err.Error()) 582 } 583 decData := r.bpool.Get(decLen) 584 decData, err = snappy.Decode(decData, data[:bh.length]) 585 r.bpool.Put(data) 586 if err != nil { 587 r.bpool.Put(decData) 588 return nil, r.newErrCorruptedBH(bh, err.Error()) 589 } 590 data = decData 591 default: 592 r.bpool.Put(data) 593 return nil, r.newErrCorruptedBH(bh, fmt.Sprintf("unknown compression type %#x", data[bh.length])) 594 } 595 return data, nil 596 } 597 598 func (r *Reader) readBlock(bh blockHandle, verifyChecksum bool) (*block, error) { 599 data, err := r.readRawBlock(bh, verifyChecksum) 600 if err != nil { 601 return nil, err 602 } 603 restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:])) 604 b := &block{ 605 bpool: r.bpool, 606 bh: bh, 607 data: data, 608 restartsLen: restartsLen, 609 restartsOffset: len(data) - (restartsLen+1)*4, 610 } 611 return b, nil 612 } 613 614 func (r *Reader) readBlockCached(bh blockHandle, verifyChecksum, fillCache bool) (*block, util.Releaser, error) { 615 if r.cache != nil { 616 var ( 617 err error 618 ch *cache.Handle 619 ) 620 if fillCache { 621 ch = r.cache.Get(bh.offset, func() (size int, value cache.Value) { 622 var b *block 623 b, err = r.readBlock(bh, verifyChecksum) 624 if err != nil { 625 return 0, nil 626 } 627 return cap(b.data), b 628 }) 629 } else { 630 ch = r.cache.Get(bh.offset, nil) 631 } 632 if ch != nil { 633 b, ok := ch.Value().(*block) 634 if !ok { 635 ch.Release() 636 return nil, nil, errors.New("leveldb/table: inconsistent block type") 637 } 638 return b, ch, err 639 } else if err != nil { 640 return nil, nil, err 641 } 642 } 643 644 b, err := r.readBlock(bh, verifyChecksum) 645 return b, b, err 646 } 647 648 func (r *Reader) readFilterBlock(bh blockHandle) (*filterBlock, error) { 649 data, err := r.readRawBlock(bh, true) 650 if err != nil { 651 return nil, err 652 } 653 n := len(data) 654 if n < 5 { 655 return nil, r.newErrCorruptedBH(bh, "too short") 656 } 657 m := n - 5 658 oOffset := int(binary.LittleEndian.Uint32(data[m:])) 659 if oOffset > m { 660 return nil, r.newErrCorruptedBH(bh, "invalid data-offsets offset") 661 } 662 b := &filterBlock{ 663 bpool: r.bpool, 664 data: data, 665 oOffset: oOffset, 666 baseLg: uint(data[n-1]), 667 filtersNum: (m - oOffset) / 4, 668 } 669 return b, nil 670 } 671 672 func (r *Reader) readFilterBlockCached(bh blockHandle, fillCache bool) (*filterBlock, util.Releaser, error) { 673 if r.cache != nil { 674 var ( 675 err error 676 ch *cache.Handle 677 ) 678 if fillCache { 679 ch = r.cache.Get(bh.offset, func() (size int, value cache.Value) { 680 var b *filterBlock 681 b, err = r.readFilterBlock(bh) 682 if err != nil { 683 return 0, nil 684 } 685 return cap(b.data), b 686 }) 687 } else { 688 ch = r.cache.Get(bh.offset, nil) 689 } 690 if ch != nil { 691 b, ok := ch.Value().(*filterBlock) 692 if !ok { 693 ch.Release() 694 return nil, nil, errors.New("leveldb/table: inconsistent block type") 695 } 696 return b, ch, err 697 } else if err != nil { 698 return nil, nil, err 699 } 700 } 701 702 b, err := r.readFilterBlock(bh) 703 return b, b, err 704 } 705 706 func (r *Reader) getIndexBlock(fillCache bool) (b *block, rel util.Releaser, err error) { 707 if r.indexBlock == nil { 708 return r.readBlockCached(r.indexBH, true, fillCache) 709 } 710 return r.indexBlock, util.NoopReleaser{}, nil 711 } 712 713 func (r *Reader) getFilterBlock(fillCache bool) (*filterBlock, util.Releaser, error) { 714 if r.filterBlock == nil { 715 return r.readFilterBlockCached(r.filterBH, fillCache) 716 } 717 return r.filterBlock, util.NoopReleaser{}, nil 718 } 719 720 func (r *Reader) newBlockIter(b *block, bReleaser util.Releaser, slice *util.Range, inclLimit bool) *blockIter { 721 bi := &blockIter{ 722 tr: r, 723 block: b, 724 blockReleaser: bReleaser, 725 // Valid key should never be nil. 726 key: make([]byte, 0), 727 dir: dirSOI, 728 riStart: 0, 729 riLimit: b.restartsLen, 730 offsetStart: 0, 731 offsetRealStart: 0, 732 offsetLimit: b.restartsOffset, 733 } 734 if slice != nil { 735 if slice.Start != nil { 736 if bi.Seek(slice.Start) { 737 bi.riStart = b.restartIndex(bi.restartIndex, b.restartsLen, bi.prevOffset) 738 bi.offsetStart = b.restartOffset(bi.riStart) 739 bi.offsetRealStart = bi.prevOffset 740 } else { 741 bi.riStart = b.restartsLen 742 bi.offsetStart = b.restartsOffset 743 bi.offsetRealStart = b.restartsOffset 744 } 745 } 746 if slice.Limit != nil { 747 if bi.Seek(slice.Limit) && (!inclLimit || bi.Next()) { 748 bi.offsetLimit = bi.prevOffset 749 bi.riLimit = bi.restartIndex + 1 750 } 751 } 752 bi.reset() 753 if bi.offsetStart > bi.offsetLimit { 754 bi.sErr(errors.New("leveldb/table: invalid slice range")) 755 } 756 } 757 return bi 758 } 759 760 func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, verifyChecksum, fillCache bool) iterator.Iterator { 761 b, rel, err := r.readBlockCached(dataBH, verifyChecksum, fillCache) 762 if err != nil { 763 return iterator.NewEmptyIterator(err) 764 } 765 return r.newBlockIter(b, rel, slice, false) 766 } 767 768 func (r *Reader) getDataIterErr(dataBH blockHandle, slice *util.Range, verifyChecksum, fillCache bool) iterator.Iterator { 769 r.mu.RLock() 770 defer r.mu.RUnlock() 771 772 if r.err != nil { 773 return iterator.NewEmptyIterator(r.err) 774 } 775 776 return r.getDataIter(dataBH, slice, verifyChecksum, fillCache) 777 } 778 779 // NewIterator creates an iterator from the table. 780 // 781 // Slice allows slicing the iterator to only contains keys in the given 782 // range. A nil Range.Start is treated as a key before all keys in the 783 // table. And a nil Range.Limit is treated as a key after all keys in 784 // the table. 785 // 786 // The returned iterator is not goroutine-safe and should be released 787 // when not used. 788 // 789 // Also read Iterator documentation of the leveldb/iterator package. 790 func (r *Reader) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { 791 r.mu.RLock() 792 defer r.mu.RUnlock() 793 794 if r.err != nil { 795 return iterator.NewEmptyIterator(r.err) 796 } 797 798 fillCache := !ro.GetDontFillCache() 799 indexBlock, rel, err := r.getIndexBlock(fillCache) 800 if err != nil { 801 return iterator.NewEmptyIterator(err) 802 } 803 index := &indexIter{ 804 blockIter: r.newBlockIter(indexBlock, rel, slice, true), 805 tr: r, 806 slice: slice, 807 fillCache: !ro.GetDontFillCache(), 808 } 809 return iterator.NewIndexedIterator(index, opt.GetStrict(r.o, ro, opt.StrictReader)) 810 } 811 812 func (r *Reader) find(key []byte, filtered bool, ro *opt.ReadOptions, noValue bool) (rkey, value []byte, err error) { 813 r.mu.RLock() 814 defer r.mu.RUnlock() 815 816 if r.err != nil { 817 err = r.err 818 return 819 } 820 821 indexBlock, rel, err := r.getIndexBlock(true) 822 if err != nil { 823 return 824 } 825 defer rel.Release() 826 827 index := r.newBlockIter(indexBlock, nil, nil, true) 828 defer index.Release() 829 if !index.Seek(key) { 830 err = index.Error() 831 if err == nil { 832 err = ErrNotFound 833 } 834 return 835 } 836 dataBH, n := decodeBlockHandle(index.Value()) 837 if n == 0 { 838 r.err = r.newErrCorruptedBH(r.indexBH, "bad data block handle") 839 return 840 } 841 if filtered && r.filter != nil { 842 filterBlock, frel, ferr := r.getFilterBlock(true) 843 if ferr == nil { 844 if !filterBlock.contains(r.filter, dataBH.offset, key) { 845 frel.Release() 846 return nil, nil, ErrNotFound 847 } 848 frel.Release() 849 } else if !errors.IsCorrupted(ferr) { 850 err = ferr 851 return 852 } 853 } 854 data := r.getDataIter(dataBH, nil, r.verifyChecksum, !ro.GetDontFillCache()) 855 defer data.Release() 856 if !data.Seek(key) { 857 err = data.Error() 858 if err == nil { 859 err = ErrNotFound 860 } 861 return 862 } 863 // Don't use block buffer, no need to copy the buffer. 864 rkey = data.Key() 865 if !noValue { 866 if r.bpool == nil { 867 value = data.Value() 868 } else { 869 // Use block buffer, and since the buffer will be recycled, the buffer 870 // need to be copied. 871 value = append([]byte{}, data.Value()...) 872 } 873 } 874 return 875 } 876 877 // Find finds key/value pair whose key is greater than or equal to the 878 // given key. It returns ErrNotFound if the table doesn't contain 879 // such pair. 880 // If filtered is true then the nearest 'block' will be checked against 881 // 'filter data' (if present) and will immediately return ErrNotFound if 882 // 'filter data' indicates that such pair doesn't exist. 883 // 884 // The caller may modify the contents of the returned slice as it is its 885 // own copy. 886 // It is safe to modify the contents of the argument after Find returns. 887 func (r *Reader) Find(key []byte, filtered bool, ro *opt.ReadOptions) (rkey, value []byte, err error) { 888 return r.find(key, filtered, ro, false) 889 } 890 891 // Find finds key that is greater than or equal to the given key. 892 // It returns ErrNotFound if the table doesn't contain such key. 893 // If filtered is true then the nearest 'block' will be checked against 894 // 'filter data' (if present) and will immediately return ErrNotFound if 895 // 'filter data' indicates that such key doesn't exist. 896 // 897 // The caller may modify the contents of the returned slice as it is its 898 // own copy. 899 // It is safe to modify the contents of the argument after Find returns. 900 func (r *Reader) FindKey(key []byte, filtered bool, ro *opt.ReadOptions) (rkey []byte, err error) { 901 rkey, _, err = r.find(key, filtered, ro, true) 902 return 903 } 904 905 // Get gets the value for the given key. It returns errors.ErrNotFound 906 // if the table does not contain the key. 907 // 908 // The caller may modify the contents of the returned slice as it is its 909 // own copy. 910 // It is safe to modify the contents of the argument after Find returns. 911 func (r *Reader) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) { 912 r.mu.RLock() 913 defer r.mu.RUnlock() 914 915 if r.err != nil { 916 err = r.err 917 return 918 } 919 920 rkey, value, err := r.find(key, false, ro, false) 921 if err == nil && r.cmp.Compare(rkey, key) != 0 { 922 value = nil 923 err = ErrNotFound 924 } 925 return 926 } 927 928 // OffsetOf returns approximate offset for the given key. 929 // 930 // It is safe to modify the contents of the argument after Get returns. 931 func (r *Reader) OffsetOf(key []byte) (offset int64, err error) { 932 r.mu.RLock() 933 defer r.mu.RUnlock() 934 935 if r.err != nil { 936 err = r.err 937 return 938 } 939 940 indexBlock, rel, err := r.readBlockCached(r.indexBH, true, true) 941 if err != nil { 942 return 943 } 944 defer rel.Release() 945 946 index := r.newBlockIter(indexBlock, nil, nil, true) 947 defer index.Release() 948 if index.Seek(key) { 949 dataBH, n := decodeBlockHandle(index.Value()) 950 if n == 0 { 951 r.err = r.newErrCorruptedBH(r.indexBH, "bad data block handle") 952 return 953 } 954 offset = int64(dataBH.offset) 955 return 956 } 957 err = index.Error() 958 if err == nil { 959 offset = r.dataEnd 960 } 961 return 962 } 963 964 // Release implements util.Releaser. 965 // It also close the file if it is an io.Closer. 966 func (r *Reader) Release() { 967 r.mu.Lock() 968 defer r.mu.Unlock() 969 970 if closer, ok := r.reader.(io.Closer); ok { 971 closer.Close() 972 } 973 if r.indexBlock != nil { 974 r.indexBlock.Release() 975 r.indexBlock = nil 976 } 977 if r.filterBlock != nil { 978 r.filterBlock.Release() 979 r.filterBlock = nil 980 } 981 r.reader = nil 982 r.cache = nil 983 r.bpool = nil 984 r.err = ErrReaderReleased 985 } 986 987 // NewReader creates a new initialized table reader for the file. 988 // The fi, cache and bpool is optional and can be nil. 989 // 990 // The returned table reader instance is goroutine-safe. 991 func NewReader(f io.ReaderAt, size int64, fd storage.FileDesc, cache *cache.NamespaceGetter, bpool *util.BufferPool, o *opt.Options) (*Reader, error) { 992 if f == nil { 993 return nil, errors.New("leveldb/table: nil file") 994 } 995 996 r := &Reader{ 997 fd: fd, 998 reader: f, 999 cache: cache, 1000 bpool: bpool, 1001 o: o, 1002 cmp: o.GetComparer(), 1003 verifyChecksum: o.GetStrict(opt.StrictBlockChecksum), 1004 } 1005 1006 if size < footerLen { 1007 r.err = r.newErrCorrupted(0, size, "table", "too small") 1008 return r, nil 1009 } 1010 1011 footerPos := size - footerLen 1012 var footer [footerLen]byte 1013 if _, err := r.reader.ReadAt(footer[:], footerPos); err != nil && err != io.EOF { 1014 return nil, err 1015 } 1016 if string(footer[footerLen-len(magic):footerLen]) != magic { 1017 r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad magic number") 1018 return r, nil 1019 } 1020 1021 var n int 1022 // Decode the metaindex block handle. 1023 r.metaBH, n = decodeBlockHandle(footer[:]) 1024 if n == 0 { 1025 r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad metaindex block handle") 1026 return r, nil 1027 } 1028 1029 // Decode the index block handle. 1030 r.indexBH, n = decodeBlockHandle(footer[n:]) 1031 if n == 0 { 1032 r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad index block handle") 1033 return r, nil 1034 } 1035 1036 // Read metaindex block. 1037 metaBlock, err := r.readBlock(r.metaBH, true) 1038 if err != nil { 1039 if errors.IsCorrupted(err) { 1040 r.err = err 1041 return r, nil 1042 } else { 1043 return nil, err 1044 } 1045 } 1046 1047 // Set data end. 1048 r.dataEnd = int64(r.metaBH.offset) 1049 1050 // Read metaindex. 1051 metaIter := r.newBlockIter(metaBlock, nil, nil, true) 1052 for metaIter.Next() { 1053 key := string(metaIter.Key()) 1054 if !strings.HasPrefix(key, "filter.") { 1055 continue 1056 } 1057 fn := key[7:] 1058 if f0 := o.GetFilter(); f0 != nil && f0.Name() == fn { 1059 r.filter = f0 1060 } else { 1061 for _, f0 := range o.GetAltFilters() { 1062 if f0.Name() == fn { 1063 r.filter = f0 1064 break 1065 } 1066 } 1067 } 1068 if r.filter != nil { 1069 filterBH, n := decodeBlockHandle(metaIter.Value()) 1070 if n == 0 { 1071 continue 1072 } 1073 r.filterBH = filterBH 1074 // Update data end. 1075 r.dataEnd = int64(filterBH.offset) 1076 break 1077 } 1078 } 1079 metaIter.Release() 1080 metaBlock.Release() 1081 1082 // Cache index and filter block locally, since we don't have global cache. 1083 if cache == nil { 1084 r.indexBlock, err = r.readBlock(r.indexBH, true) 1085 if err != nil { 1086 if errors.IsCorrupted(err) { 1087 r.err = err 1088 return r, nil 1089 } else { 1090 return nil, err 1091 } 1092 } 1093 if r.filter != nil { 1094 r.filterBlock, err = r.readFilterBlock(r.filterBH) 1095 if err != nil { 1096 if !errors.IsCorrupted(err) { 1097 return nil, err 1098 } 1099 1100 // Don't use filter then. 1101 r.filter = nil 1102 } 1103 } 1104 } 1105 1106 return r, nil 1107 }