github.com/petermattis/pebble@v0.0.0-20190905164901-ab51a2166067/iterator_test.go (about) 1 // Copyright 2013 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 // of this source code is governed by a BSD-style license that can be found in 3 // the LICENSE file. 4 5 package pebble 6 7 import ( 8 "bytes" 9 "errors" 10 "fmt" 11 "strconv" 12 "strings" 13 "testing" 14 "time" 15 16 "github.com/petermattis/pebble/internal/base" 17 "github.com/petermattis/pebble/internal/datadriven" 18 "golang.org/x/exp/rand" 19 ) 20 21 var testKeyValuePairs = []string{ 22 "10:10", 23 "11:11", 24 "12:12", 25 "13:13", 26 "14:14", 27 "15:15", 28 "16:16", 29 "17:17", 30 "18:18", 31 "19:19", 32 } 33 34 type fakeIter struct { 35 lower []byte 36 upper []byte 37 keys []InternalKey 38 vals [][]byte 39 index int 40 valid bool 41 closeErr error 42 } 43 44 func fakeIkey(s string) InternalKey { 45 j := strings.Index(s, ":") 46 seqNum, err := strconv.Atoi(s[j+1:]) 47 if err != nil { 48 panic(err) 49 } 50 return base.MakeInternalKey([]byte(s[:j]), uint64(seqNum), InternalKeyKindSet) 51 } 52 53 func newFakeIterator(closeErr error, keys ...string) *fakeIter { 54 ikeys := make([]InternalKey, len(keys)) 55 for i, k := range keys { 56 ikeys[i] = fakeIkey(k) 57 } 58 return &fakeIter{ 59 keys: ikeys, 60 index: 0, 61 valid: len(ikeys) > 0, 62 closeErr: closeErr, 63 } 64 } 65 66 func (f *fakeIter) SeekGE(key []byte) (*InternalKey, []byte) { 67 f.valid = false 68 for f.index = 0; f.index < len(f.keys); f.index++ { 69 if DefaultComparer.Compare(key, f.Key().UserKey) <= 0 { 70 if f.upper != nil && DefaultComparer.Compare(f.upper, f.Key().UserKey) <= 0 { 71 return nil, nil 72 } 73 f.valid = true 74 return f.Key(), f.Value() 75 } 76 } 77 return nil, nil 78 } 79 80 func (f *fakeIter) SeekPrefixGE(prefix, key []byte) (*InternalKey, []byte) { 81 return f.SeekGE(key) 82 } 83 84 func (f *fakeIter) SeekLT(key []byte) (*InternalKey, []byte) { 85 f.valid = false 86 for f.index = len(f.keys) - 1; f.index >= 0; f.index-- { 87 if DefaultComparer.Compare(key, f.Key().UserKey) > 0 { 88 if f.lower != nil && DefaultComparer.Compare(f.lower, f.Key().UserKey) > 0 { 89 return nil, nil 90 } 91 f.valid = true 92 return f.Key(), f.Value() 93 } 94 } 95 return nil, nil 96 } 97 98 func (f *fakeIter) First() (*InternalKey, []byte) { 99 f.valid = false 100 f.index = -1 101 if key, _ := f.Next(); key == nil { 102 return nil, nil 103 } 104 if f.upper != nil && DefaultComparer.Compare(f.upper, f.Key().UserKey) <= 0 { 105 return nil, nil 106 } 107 f.valid = true 108 return f.Key(), f.Value() 109 } 110 111 func (f *fakeIter) Last() (*InternalKey, []byte) { 112 f.valid = false 113 f.index = len(f.keys) 114 if key, _ := f.Prev(); key == nil { 115 return nil, nil 116 } 117 if f.lower != nil && DefaultComparer.Compare(f.lower, f.Key().UserKey) > 0 { 118 return nil, nil 119 } 120 f.valid = true 121 return f.Key(), f.Value() 122 } 123 124 func (f *fakeIter) Next() (*InternalKey, []byte) { 125 f.valid = false 126 if f.index == len(f.keys) { 127 return nil, nil 128 } 129 f.index++ 130 if f.index == len(f.keys) { 131 return nil, nil 132 } 133 if f.upper != nil && DefaultComparer.Compare(f.upper, f.Key().UserKey) <= 0 { 134 return nil, nil 135 } 136 f.valid = true 137 return f.Key(), f.Value() 138 } 139 140 func (f *fakeIter) Prev() (*InternalKey, []byte) { 141 f.valid = false 142 if f.index < 0 { 143 return nil, nil 144 } 145 f.index-- 146 if f.index < 0 { 147 return nil, nil 148 } 149 if f.lower != nil && DefaultComparer.Compare(f.lower, f.Key().UserKey) > 0 { 150 return nil, nil 151 } 152 f.valid = true 153 return f.Key(), f.Value() 154 } 155 156 func (f *fakeIter) Key() *InternalKey { 157 if f.index >= 0 && f.index < len(f.keys) { 158 return &f.keys[f.index] 159 } 160 return nil 161 } 162 163 func (f *fakeIter) Value() []byte { 164 if f.index >= 0 && f.index < len(f.vals) { 165 return f.vals[f.index] 166 } 167 return nil 168 } 169 170 func (f *fakeIter) Valid() bool { 171 return f.index >= 0 && f.index < len(f.keys) && f.valid 172 } 173 174 func (f *fakeIter) Error() error { 175 return f.closeErr 176 } 177 178 func (f *fakeIter) Close() error { 179 return f.closeErr 180 } 181 182 func (f *fakeIter) SetBounds(lower, upper []byte) { 183 f.lower = lower 184 f.upper = upper 185 } 186 187 // testIterator tests creating a combined iterator from a number of sub- 188 // iterators. newFunc is a constructor function. splitFunc returns a random 189 // split of the testKeyValuePairs slice such that walking a combined iterator 190 // over those splits should recover the original key/value pairs in order. 191 func testIterator( 192 t *testing.T, 193 newFunc func(...internalIterator) internalIterator, 194 splitFunc func(r *rand.Rand) [][]string, 195 ) { 196 // Test pre-determined sub-iterators. The sub-iterators are designed 197 // so that the combined key/value pair order is the same whether the 198 // combined iterator is concatenating or merging. 199 testCases := []struct { 200 desc string 201 iters []internalIterator 202 want string 203 }{ 204 { 205 "one sub-iterator", 206 []internalIterator{ 207 newFakeIterator(nil, "e:1", "w:2"), 208 }, 209 "<e:1><w:2>.", 210 }, 211 { 212 "two sub-iterators", 213 []internalIterator{ 214 newFakeIterator(nil, "a0:0"), 215 newFakeIterator(nil, "b1:1", "b2:2"), 216 }, 217 "<a0:0><b1:1><b2:2>.", 218 }, 219 { 220 "empty sub-iterators", 221 []internalIterator{ 222 newFakeIterator(nil), 223 newFakeIterator(nil), 224 newFakeIterator(nil), 225 }, 226 ".", 227 }, 228 { 229 "sub-iterator errors", 230 []internalIterator{ 231 newFakeIterator(nil, "a0:0", "a1:1"), 232 newFakeIterator(errors.New("the sky is falling!"), "b2:2", "b3:3", "b4:4"), 233 newFakeIterator(errors.New("run for your lives!"), "c5:5", "c6:6"), 234 }, 235 "<a0:0><a1:1><b2:2><b3:3><b4:4>err=the sky is falling!", 236 }, 237 } 238 for _, tc := range testCases { 239 var b bytes.Buffer 240 iter := newFunc(tc.iters...) 241 for key, _ := iter.First(); key != nil; key, _ = iter.Next() { 242 fmt.Fprintf(&b, "<%s:%d>", key.UserKey, key.SeqNum()) 243 } 244 if err := iter.Close(); err != nil { 245 fmt.Fprintf(&b, "err=%v", err) 246 } else { 247 b.WriteByte('.') 248 } 249 if got := b.String(); got != tc.want { 250 t.Errorf("%s:\ngot %q\nwant %q", tc.desc, got, tc.want) 251 } 252 } 253 254 // Test randomly generated sub-iterators. 255 r := rand.New(rand.NewSource(0)) 256 for i, nBad := 0, 0; i < 1000; i++ { 257 bad := false 258 259 splits := splitFunc(r) 260 iters := make([]internalIterator, len(splits)) 261 for i, split := range splits { 262 iters[i] = newFakeIterator(nil, split...) 263 } 264 iter := newFunc(iters...) 265 266 j := 0 267 for ; iter.Valid() && j < len(testKeyValuePairs); j++ { 268 got := fmt.Sprintf("%s:%d", iter.Key().UserKey, iter.Key().SeqNum()) 269 want := testKeyValuePairs[j] 270 if got != want { 271 bad = true 272 t.Errorf("random splits: i=%d, j=%d: got %q, want %q", i, j, got, want) 273 } 274 iter.Next() 275 } 276 if iter.Valid() { 277 bad = true 278 t.Errorf("random splits: i=%d, j=%d: iter was not exhausted", i, j) 279 } 280 if j != len(testKeyValuePairs) { 281 bad = true 282 t.Errorf("random splits: i=%d, j=%d: want j=%d", i, j, len(testKeyValuePairs)) 283 fmt.Printf("splits: %v\n", splits) 284 return 285 } 286 if err := iter.Close(); err != nil { 287 bad = true 288 t.Errorf("random splits: i=%d, j=%d: %v", i, j, err) 289 } 290 291 if bad { 292 nBad++ 293 if nBad == 10 { 294 t.Fatal("random splits: too many errors; stopping") 295 } 296 } 297 } 298 } 299 300 func TestIterator(t *testing.T) { 301 var keys []InternalKey 302 var vals [][]byte 303 304 newIter := func(seqNum uint64, opts IterOptions) *Iterator { 305 cmp := DefaultComparer.Compare 306 equal := DefaultComparer.Equal 307 split := func(a []byte) int { return len(a) } 308 // NB: Use a mergingIter to filter entries newer than seqNum. 309 iter := newMergingIter(cmp, &fakeIter{ 310 lower: opts.GetLowerBound(), 311 upper: opts.GetUpperBound(), 312 keys: keys, 313 vals: vals, 314 }) 315 iter.snapshot = seqNum 316 return &Iterator{ 317 opts: opts, 318 cmp: cmp, 319 equal: equal, 320 split: split, 321 merge: DefaultMerger.Merge, 322 iter: iter, 323 } 324 } 325 326 datadriven.RunTest(t, "testdata/iterator", func(d *datadriven.TestData) string { 327 switch d.Cmd { 328 case "define": 329 keys = keys[:0] 330 vals = vals[:0] 331 for _, key := range strings.Split(d.Input, "\n") { 332 j := strings.Index(key, ":") 333 keys = append(keys, base.ParseInternalKey(key[:j])) 334 vals = append(vals, []byte(key[j+1:])) 335 } 336 return "" 337 338 case "iter": 339 var seqNum int 340 var opts IterOptions 341 342 for _, arg := range d.CmdArgs { 343 if len(arg.Vals) != 1 { 344 return fmt.Sprintf("%s: %s=<value>", d.Cmd, arg.Key) 345 } 346 switch arg.Key { 347 case "seq": 348 var err error 349 seqNum, err = strconv.Atoi(arg.Vals[0]) 350 if err != nil { 351 return err.Error() 352 } 353 case "lower": 354 opts.LowerBound = []byte(arg.Vals[0]) 355 case "upper": 356 opts.UpperBound = []byte(arg.Vals[0]) 357 default: 358 return fmt.Sprintf("%s: unknown arg: %s", d.Cmd, arg.Key) 359 } 360 } 361 362 iter := newIter(uint64(seqNum), opts) 363 defer iter.Close() 364 return runIterCmd(d, iter) 365 366 default: 367 return fmt.Sprintf("unknown command: %s", d.Cmd) 368 } 369 }) 370 } 371 372 type minSeqNumPropertyCollector struct { 373 minSeqNum uint64 374 } 375 376 func (c *minSeqNumPropertyCollector) Add(key InternalKey, value []byte) error { 377 if c.minSeqNum == 0 || c.minSeqNum > key.SeqNum() { 378 c.minSeqNum = key.SeqNum() 379 } 380 return nil 381 } 382 383 func (c *minSeqNumPropertyCollector) Finish(userProps map[string]string) error { 384 userProps["test.min-seq-num"] = fmt.Sprint(c.minSeqNum) 385 return nil 386 } 387 388 func (c *minSeqNumPropertyCollector) Name() string { 389 return "minSeqNumPropertyCollector" 390 } 391 392 func TestIteratorTableFilter(t *testing.T) { 393 var d *DB 394 395 datadriven.RunTest(t, "testdata/iterator_table_filter", func(td *datadriven.TestData) string { 396 switch td.Cmd { 397 case "define": 398 opts := &Options{} 399 opts.TablePropertyCollectors = append(opts.TablePropertyCollectors, 400 func() TablePropertyCollector { 401 return &minSeqNumPropertyCollector{} 402 }) 403 404 var err error 405 if d, err = runDBDefineCmd(td, opts); err != nil { 406 return err.Error() 407 } 408 409 d.mu.Lock() 410 // Disable the "dynamic base level" code for this test. 411 d.mu.versions.picker.baseLevel = 1 412 s := d.mu.versions.currentVersion().DebugString() 413 d.mu.Unlock() 414 return s 415 416 case "iter": 417 // We're using an iterator table filter to approximate what is done by 418 // snapshots. 419 iterOpts := &IterOptions{} 420 for _, arg := range td.CmdArgs { 421 if len(arg.Vals) != 1 { 422 return fmt.Sprintf("%s: %s=<value>", td.Cmd, arg.Key) 423 } 424 switch arg.Key { 425 case "filter": 426 seqNum, err := strconv.ParseUint(arg.Vals[0], 10, 64) 427 if err != nil { 428 return err.Error() 429 } 430 iterOpts.TableFilter = func(userProps map[string]string) bool { 431 minSeqNum, err := strconv.ParseUint(userProps["test.min-seq-num"], 10, 64) 432 if err != nil { 433 return true 434 } 435 return minSeqNum < seqNum 436 } 437 default: 438 return fmt.Sprintf("%s: unknown arg: %s", td.Cmd, arg.Key) 439 } 440 } 441 442 // TODO(peter): runDBDefineCmd doesn't properly update the visible 443 // sequence number. So we have to use a snapshot with a very large 444 // sequence number, otherwise the DB appears empty. 445 snap := Snapshot{ 446 db: d, 447 seqNum: InternalKeySeqNumMax, 448 } 449 iter := snap.NewIter(iterOpts) 450 defer iter.Close() 451 return runIterCmd(td, iter) 452 453 default: 454 return fmt.Sprintf("unknown command: %s", td.Cmd) 455 } 456 }) 457 } 458 459 func BenchmarkIteratorSeekGE(b *testing.B) { 460 m, keys := buildMemTable(b) 461 iter := &Iterator{ 462 cmp: DefaultComparer.Compare, 463 equal: DefaultComparer.Equal, 464 iter: m.newIter(nil), 465 } 466 rng := rand.New(rand.NewSource(uint64(time.Now().UnixNano()))) 467 468 b.ResetTimer() 469 for i := 0; i < b.N; i++ { 470 key := keys[rng.Intn(len(keys))] 471 iter.SeekGE(key) 472 } 473 } 474 475 func BenchmarkIteratorNext(b *testing.B) { 476 m, _ := buildMemTable(b) 477 iter := &Iterator{ 478 cmp: DefaultComparer.Compare, 479 equal: DefaultComparer.Equal, 480 iter: m.newIter(nil), 481 } 482 483 b.ResetTimer() 484 for i := 0; i < b.N; i++ { 485 if !iter.Valid() { 486 iter.First() 487 } 488 iter.Next() 489 } 490 } 491 492 func BenchmarkIteratorPrev(b *testing.B) { 493 m, _ := buildMemTable(b) 494 iter := &Iterator{ 495 cmp: DefaultComparer.Compare, 496 equal: DefaultComparer.Equal, 497 iter: m.newIter(nil), 498 } 499 500 b.ResetTimer() 501 for i := 0; i < b.N; i++ { 502 if !iter.Valid() { 503 iter.Last() 504 } 505 iter.Prev() 506 } 507 }