github.com/coocood/badger@v1.5.1-0.20200528065104-c02ac3616d04/iterator.go (about) 1 /* 2 * Copyright 2017 Dgraph Labs, Inc. and Contributors 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package badger 18 19 import ( 20 "bytes" 21 "fmt" 22 "math" 23 "sort" 24 "sync/atomic" 25 26 "github.com/coocood/badger/table" 27 "github.com/coocood/badger/table/memtable" 28 "github.com/coocood/badger/y" 29 "github.com/dgryski/go-farm" 30 ) 31 32 // Item is returned during iteration. Both the Key() and Value() output is only valid until 33 // iterator.Next() is called. 34 type Item struct { 35 err error 36 db *DB 37 key y.Key 38 vptr []byte 39 meta byte // We need to store meta to know about bitValuePointer. 40 userMeta []byte 41 slice *y.Slice 42 next *Item 43 txn *Txn 44 } 45 46 // String returns a string representation of Item 47 func (item *Item) String() string { 48 return fmt.Sprintf("key=%q, version=%d, meta=%x", item.Key(), item.Version(), item.meta) 49 } 50 51 // Key returns the key. 52 // 53 // Key is only valid as long as item is valid, or transaction is valid. If you need to use it 54 // outside its validity, please use KeyCopy 55 func (item *Item) Key() []byte { 56 return item.key.UserKey 57 } 58 59 // KeyCopy returns a copy of the key of the item, writing it to dst slice. 60 // If nil is passed, or capacity of dst isn't sufficient, a new slice would be allocated and 61 // returned. 62 func (item *Item) KeyCopy(dst []byte) []byte { 63 return y.SafeCopy(dst, item.key.UserKey) 64 } 65 66 // Version returns the commit timestamp of the item. 67 func (item *Item) Version() uint64 { 68 return item.key.Version 69 } 70 71 // IsEmpty checks if the value is empty. 72 func (item *Item) IsEmpty() bool { 73 return len(item.vptr) == 0 74 } 75 76 // Value retrieves the value of the item from the value log. 77 // 78 // This method must be called within a transaction. Calling it outside a 79 // transaction is considered undefined behavior. If an iterator is being used, 80 // then Item.Value() is defined in the current iteration only, because items are 81 // reused. 82 // 83 // If you need to use a value outside a transaction, please use Item.ValueCopy 84 // instead, or copy it yourself. Value might change once discard or commit is called. 85 // Use ValueCopy if you want to do a Set after Get. 86 func (item *Item) Value() ([]byte, error) { 87 if item.meta&bitValuePointer > 0 { 88 if item.slice == nil { 89 item.slice = new(y.Slice) 90 } 91 if item.txn.blobCache == nil { 92 item.txn.blobCache = map[uint32]*blobCache{} 93 } 94 return item.db.blobManger.read(item.vptr, item.slice, item.txn.blobCache) 95 } 96 return item.vptr, nil 97 } 98 99 // ValueSize returns the size of the value without the cost of retrieving the value. 100 func (item *Item) ValueSize() int { 101 if item.meta&bitValuePointer > 0 { 102 var bp blobPointer 103 bp.decode(item.vptr) 104 return int(bp.length) 105 } 106 return len(item.vptr) 107 } 108 109 // ValueCopy returns a copy of the value of the item from the value log, writing it to dst slice. 110 // If nil is passed, or capacity of dst isn't sufficient, a new slice would be allocated and 111 // returned. Tip: It might make sense to reuse the returned slice as dst argument for the next call. 112 // 113 // This function is useful in long running iterate/update transactions to avoid a write deadlock. 114 // See Github issue: https://github.com/coocood/badger/issues/315 115 func (item *Item) ValueCopy(dst []byte) ([]byte, error) { 116 buf, err := item.Value() 117 if err != nil { 118 return nil, err 119 } 120 return y.SafeCopy(dst, buf), nil 121 } 122 123 func (item *Item) hasValue() bool { 124 if item.meta == 0 && item.vptr == nil { 125 // key not found 126 return false 127 } 128 return true 129 } 130 131 // IsDeleted returns true if item contains deleted or expired value. 132 func (item *Item) IsDeleted() bool { 133 return isDeleted(item.meta) 134 } 135 136 // EstimatedSize returns approximate size of the key-value pair. 137 // 138 // This can be called while iterating through a store to quickly estimate the 139 // size of a range of key-value pairs (without fetching the corresponding 140 // values). 141 func (item *Item) EstimatedSize() int64 { 142 if !item.hasValue() { 143 return 0 144 } 145 return int64(item.key.Len() + len(item.vptr)) 146 } 147 148 // UserMeta returns the userMeta set by the user. Typically, this byte, optionally set by the user 149 // is used to interpret the value. 150 func (item *Item) UserMeta() []byte { 151 return item.userMeta 152 } 153 154 // IteratorOptions is used to set options when iterating over Badger key-value 155 // stores. 156 // 157 // This package provides DefaultIteratorOptions which contains options that 158 // should work for most applications. Consider using that as a starting point 159 // before customizing it for your own needs. 160 type IteratorOptions struct { 161 Reverse bool // Direction of iteration. False is forward, true is backward. 162 AllVersions bool // Fetch all valid versions of the same key. 163 164 // StartKey and EndKey are used to prune non-overlapping table iterators. 165 // They are not boundary limits, the EndKey is exclusive. 166 StartKey y.Key 167 EndKey y.Key 168 169 internalAccess bool // Used to allow internal access to badger keys. 170 } 171 172 func (opts *IteratorOptions) hasRange() bool { 173 return !opts.StartKey.IsEmpty() && !opts.EndKey.IsEmpty() 174 } 175 176 func (opts *IteratorOptions) OverlapPending(it *pendingWritesIterator) bool { 177 if it == nil { 178 return false 179 } 180 if !opts.hasRange() { 181 return true 182 } 183 if opts.EndKey.Compare(it.entries[0].Key) <= 0 { 184 return false 185 } 186 if opts.StartKey.Compare(it.entries[len(it.entries)-1].Key) > 0 { 187 return false 188 } 189 return true 190 } 191 192 func (opts *IteratorOptions) OverlapMemTable(t *memtable.Table) bool { 193 if t.Empty() { 194 return false 195 } 196 if !opts.hasRange() { 197 return true 198 } 199 iter := t.NewIterator(false) 200 iter.Seek(opts.StartKey.UserKey) 201 if !iter.Valid() { 202 return false 203 } 204 if bytes.Compare(iter.Key().UserKey, opts.EndKey.UserKey) >= 0 { 205 return false 206 } 207 return true 208 } 209 210 func (opts *IteratorOptions) OverlapTable(t table.Table) bool { 211 if !opts.hasRange() { 212 return true 213 } 214 return t.HasOverlap(opts.StartKey, opts.EndKey, false) 215 } 216 217 func (opts *IteratorOptions) OverlapTables(tables []table.Table) []table.Table { 218 if len(tables) == 0 { 219 return nil 220 } 221 if !opts.hasRange() { 222 return tables 223 } 224 startIdx := sort.Search(len(tables), func(i int) bool { 225 t := tables[i] 226 return opts.StartKey.Compare(t.Biggest()) <= 0 227 }) 228 if startIdx == len(tables) { 229 return nil 230 } 231 tables = tables[startIdx:] 232 endIdx := sort.Search(len(tables), func(i int) bool { 233 t := tables[i] 234 return t.Smallest().Compare(opts.EndKey) >= 0 235 }) 236 tables = tables[:endIdx] 237 overlapTables := make([]table.Table, 0, 8) 238 for _, t := range tables { 239 if opts.OverlapTable(t) { 240 overlapTables = append(overlapTables, t) 241 } 242 } 243 return overlapTables 244 } 245 246 // DefaultIteratorOptions contains default options when iterating over Badger key-value stores. 247 var DefaultIteratorOptions = IteratorOptions{ 248 Reverse: false, 249 AllVersions: false, 250 } 251 252 // Iterator helps iterating over the KV pairs in a lexicographically sorted order. 253 type Iterator struct { 254 iitr y.Iterator 255 txn *Txn 256 readTs uint64 257 258 opt IteratorOptions 259 item *Item 260 itBuf Item 261 vs y.ValueStruct 262 } 263 264 // NewIterator returns a new iterator. Depending upon the options, either only keys, or both 265 // key-value pairs would be fetched. The keys are returned in lexicographically sorted order. 266 // Avoid long running iterations in update transactions. 267 func (txn *Txn) NewIterator(opt IteratorOptions) *Iterator { 268 atomic.AddInt32(&txn.numIterators, 1) 269 270 tables := txn.db.getMemTables() 271 if !opt.StartKey.IsEmpty() { 272 opt.StartKey.Version = math.MaxUint64 273 } 274 if !opt.EndKey.IsEmpty() { 275 opt.EndKey.Version = math.MaxUint64 276 } 277 var iters []y.Iterator 278 if itr := txn.newPendingWritesIterator(opt.Reverse); opt.OverlapPending(itr) { 279 iters = append(iters, itr) 280 } 281 for i := 0; i < len(tables); i++ { 282 if opt.OverlapMemTable(tables[i]) { 283 iters = append(iters, tables[i].NewIterator(opt.Reverse)) 284 } 285 } 286 iters = txn.db.lc.appendIterators(iters, &opt) // This will increment references. 287 res := &Iterator{ 288 txn: txn, 289 iitr: table.NewMergeIterator(iters, opt.Reverse), 290 opt: opt, 291 readTs: txn.readTs, 292 } 293 res.itBuf.db = txn.db 294 res.itBuf.txn = txn 295 res.itBuf.slice = new(y.Slice) 296 return res 297 } 298 299 // Item returns pointer to the current key-value pair. 300 // This item is only valid until it.Next() gets called. 301 func (it *Iterator) Item() *Item { 302 tx := it.txn 303 if tx.update { 304 // Track reads if this is an update txn. 305 tx.reads = append(tx.reads, farm.Fingerprint64(it.item.Key())) 306 } 307 return it.item 308 } 309 310 // Valid returns false when iteration is done. 311 func (it *Iterator) Valid() bool { return it.item != nil } 312 313 // ValidForPrefix returns false when iteration is done 314 // or when the current key is not prefixed by the specified prefix. 315 func (it *Iterator) ValidForPrefix(prefix []byte) bool { 316 return it.item != nil && bytes.HasPrefix(it.item.key.UserKey, prefix) 317 } 318 319 // Close would close the iterator. It is important to call this when you're done with iteration. 320 func (it *Iterator) Close() { 321 atomic.AddInt32(&it.txn.numIterators, -1) 322 } 323 324 // Next would advance the iterator by one. Always check it.Valid() after a Next() 325 // to ensure you have access to a valid it.Item(). 326 func (it *Iterator) Next() { 327 if it.opt.AllVersions && it.Valid() && it.iitr.NextVersion() { 328 it.updateItem() 329 return 330 } 331 it.iitr.Next() 332 it.parseItem() 333 return 334 } 335 336 func (it *Iterator) updateItem() { 337 it.iitr.FillValue(&it.vs) 338 item := &it.itBuf 339 item.key = it.iitr.Key() 340 item.meta = it.vs.Meta 341 item.userMeta = it.vs.UserMeta 342 item.vptr = it.vs.Value 343 it.item = item 344 } 345 346 func (it *Iterator) parseItem() { 347 iitr := it.iitr 348 for iitr.Valid() { 349 key := iitr.Key() 350 if !it.opt.internalAccess && key.UserKey[0] == '!' { 351 iitr.Next() 352 continue 353 } 354 if key.Version > it.readTs { 355 if !y.SeekToVersion(iitr, it.readTs) { 356 iitr.Next() 357 continue 358 } 359 } 360 it.updateItem() 361 if !it.opt.AllVersions && isDeleted(it.vs.Meta) { 362 iitr.Next() 363 continue 364 } 365 return 366 } 367 it.item = nil 368 } 369 370 func isDeleted(meta byte) bool { 371 return meta&bitDelete > 0 372 } 373 374 // Seek would seek to the provided key if present. If absent, it would seek to the next smallest key 375 // greater than provided if iterating in the forward direction. Behavior would be reversed is 376 // iterating backwards. 377 func (it *Iterator) Seek(key []byte) { 378 if !it.opt.Reverse { 379 it.iitr.Seek(key) 380 } else { 381 if len(key) == 0 { 382 it.iitr.Rewind() 383 } else { 384 it.iitr.Seek(key) 385 } 386 } 387 it.parseItem() 388 } 389 390 // Rewind would rewind the iterator cursor all the way to zero-th position, which would be the 391 // smallest key if iterating forward, and largest if iterating backward. It does not keep track of 392 // whether the cursor started with a Seek(). 393 func (it *Iterator) Rewind() { 394 it.iitr.Rewind() 395 it.parseItem() 396 } 397 398 func (it *Iterator) SetAllVersions(allVersions bool) { 399 it.opt.AllVersions = allVersions 400 }