github.com/cockroachdb/pebble@v1.1.1-0.20240513155919-3622ade60459/iterator_histories_test.go (about) 1 // Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 // of this source code is governed by a BSD-style license that can be found in 3 // the LICENSE file. 4 package pebble 5 6 import ( 7 "bytes" 8 "fmt" 9 "path/filepath" 10 "strconv" 11 "strings" 12 "testing" 13 14 "github.com/cockroachdb/datadriven" 15 "github.com/cockroachdb/errors" 16 "github.com/cockroachdb/pebble/bloom" 17 "github.com/cockroachdb/pebble/internal/base" 18 "github.com/cockroachdb/pebble/internal/invariants" 19 "github.com/cockroachdb/pebble/internal/testkeys" 20 "github.com/cockroachdb/pebble/sstable" 21 "github.com/cockroachdb/pebble/vfs" 22 "github.com/stretchr/testify/require" 23 ) 24 25 // TODO(jackson): Add a range keys test with concurrency: the logic to cache 26 // fragmented spans is susceptible to races. 27 28 func TestIterHistories(t *testing.T) { 29 datadriven.Walk(t, "testdata/iter_histories", func(t *testing.T, path string) { 30 filename := filepath.Base(path) 31 switch { 32 case invariants.Enabled && strings.Contains(filename, "no_invariants"): 33 t.Skip("disabled when run with -tags invariants due to nondeterminism") 34 } 35 36 var d *DB 37 iters := map[string]*Iterator{} 38 batches := map[string]*Batch{} 39 newIter := func(name string, reader Reader, o *IterOptions) *Iterator { 40 it, _ := reader.NewIter(o) 41 iters[name] = it 42 return it 43 } 44 parseOpts := func(td *datadriven.TestData) (*Options, error) { 45 opts := &Options{ 46 FS: vfs.NewMem(), 47 Comparer: testkeys.Comparer, 48 FormatMajorVersion: FormatRangeKeys, 49 BlockPropertyCollectors: []func() BlockPropertyCollector{ 50 sstable.NewTestKeysBlockPropertyCollector, 51 }, 52 } 53 opts.DisableAutomaticCompactions = true 54 opts.EnsureDefaults() 55 opts.WithFSDefaults() 56 57 for _, cmdArg := range td.CmdArgs { 58 switch cmdArg.Key { 59 case "format-major-version": 60 v, err := strconv.Atoi(cmdArg.Vals[0]) 61 if err != nil { 62 return nil, err 63 } 64 // Override the DB version. 65 opts.FormatMajorVersion = FormatMajorVersion(v) 66 case "block-size": 67 v, err := strconv.Atoi(cmdArg.Vals[0]) 68 if err != nil { 69 return nil, err 70 } 71 for i := range opts.Levels { 72 opts.Levels[i].BlockSize = v 73 } 74 case "index-block-size": 75 v, err := strconv.Atoi(cmdArg.Vals[0]) 76 if err != nil { 77 return nil, err 78 } 79 for i := range opts.Levels { 80 opts.Levels[i].IndexBlockSize = v 81 } 82 case "target-file-size": 83 v, err := strconv.Atoi(cmdArg.Vals[0]) 84 if err != nil { 85 return nil, err 86 } 87 for i := range opts.Levels { 88 opts.Levels[i].TargetFileSize = int64(v) 89 } 90 case "bloom-bits-per-key": 91 v, err := strconv.Atoi(cmdArg.Vals[0]) 92 if err != nil { 93 return nil, err 94 } 95 fp := bloom.FilterPolicy(v) 96 opts.Filters = map[string]FilterPolicy{fp.Name(): fp} 97 for i := range opts.Levels { 98 opts.Levels[i].FilterPolicy = fp 99 } 100 case "merger": 101 switch cmdArg.Vals[0] { 102 case "appender": 103 opts.Merger = base.DefaultMerger 104 default: 105 return nil, errors.Newf("unrecognized Merger %q\n", cmdArg.Vals[0]) 106 } 107 } 108 } 109 return opts, nil 110 } 111 cleanup := func() (err error) { 112 for key, batch := range batches { 113 err = firstError(err, batch.Close()) 114 delete(batches, key) 115 } 116 for key, iter := range iters { 117 err = firstError(err, iter.Close()) 118 delete(iters, key) 119 } 120 if d != nil { 121 err = firstError(err, d.Close()) 122 d = nil 123 } 124 return err 125 } 126 defer cleanup() 127 128 datadriven.RunTest(t, path, func(t *testing.T, td *datadriven.TestData) string { 129 switch td.Cmd { 130 case "define": 131 if err := cleanup(); err != nil { 132 return err.Error() 133 } 134 opts, err := parseOpts(td) 135 if err != nil { 136 return err.Error() 137 } 138 d, err = runDBDefineCmd(td, opts) 139 if err != nil { 140 return err.Error() 141 } 142 return runLSMCmd(td, d) 143 144 case "reset": 145 if err := cleanup(); err != nil { 146 return err.Error() 147 } 148 opts, err := parseOpts(td) 149 if err != nil { 150 return err.Error() 151 } 152 153 d, err = Open("", opts) 154 require.NoError(t, err) 155 return "" 156 case "populate": 157 b := d.NewBatch() 158 runPopulateCmd(t, td, b) 159 count := b.Count() 160 require.NoError(t, b.Commit(nil)) 161 return fmt.Sprintf("wrote %d keys\n", count) 162 case "batch": 163 var name string 164 td.MaybeScanArgs(t, "name", &name) 165 commit := td.HasArg("commit") 166 b := d.NewIndexedBatch() 167 require.NoError(t, runBatchDefineCmd(td, b)) 168 var err error 169 if commit { 170 func() { 171 defer func() { 172 if r := recover(); r != nil { 173 err = errors.New(r.(string)) 174 } 175 }() 176 err = b.Commit(nil) 177 }() 178 } else if name != "" { 179 batches[name] = b 180 } 181 if err != nil { 182 return err.Error() 183 } 184 count := b.Count() 185 if commit { 186 return fmt.Sprintf("committed %d keys\n", count) 187 } 188 return fmt.Sprintf("wrote %d keys to batch %q\n", count, name) 189 case "compact": 190 if err := runCompactCmd(td, d); err != nil { 191 return err.Error() 192 } 193 return runLSMCmd(td, d) 194 case "flush": 195 err := d.Flush() 196 if err != nil { 197 return err.Error() 198 } 199 return "" 200 case "get": 201 var reader Reader = d 202 if arg, ok := td.Arg("reader"); ok { 203 if reader, ok = batches[arg.Vals[0]]; !ok { 204 return fmt.Sprintf("unknown reader %q", arg.Vals[0]) 205 } 206 } 207 var buf bytes.Buffer 208 for _, l := range strings.Split(td.Input, "\n") { 209 v, closer, err := reader.Get([]byte(l)) 210 if err != nil { 211 fmt.Fprintf(&buf, "%s: error: %s\n", l, err) 212 } else { 213 fmt.Fprintf(&buf, "%s: %s\n", l, v) 214 } 215 if err := closer.Close(); err != nil { 216 fmt.Fprintf(&buf, "close err: %s\n", err) 217 } 218 } 219 return buf.String() 220 case "ingest": 221 if err := runBuildCmd(td, d, d.opts.FS); err != nil { 222 return err.Error() 223 } 224 if err := runIngestCmd(td, d, d.opts.FS); err != nil { 225 return err.Error() 226 } 227 return "" 228 case "lsm": 229 return runLSMCmd(td, d) 230 case "metrics": 231 d.mu.Lock() 232 d.waitTableStats() 233 d.mu.Unlock() 234 m := d.Metrics() 235 return fmt.Sprintf("Metrics.Keys.RangeKeySetsCount = %d\n", m.Keys.RangeKeySetsCount) 236 case "mutate": 237 var batchName string 238 td.ScanArgs(t, "batch", &batchName) 239 mut := newBatch(d) 240 if err := runBatchDefineCmd(td, mut); err != nil { 241 return err.Error() 242 } 243 if err := batches[batchName].Apply(mut, nil); err != nil { 244 return err.Error() 245 } 246 return "" 247 case "clone": 248 var from, to string 249 var cloneOpts CloneOptions 250 var iterOpts IterOptions 251 td.ScanArgs(t, "from", &from) 252 td.ScanArgs(t, "to", &to) 253 td.ScanArgs(t, "refresh-batch", &cloneOpts.RefreshBatchView) 254 fromIter := iters[from] 255 if foundAny, err := parseIterOptions(&iterOpts, &fromIter.opts, strings.Fields(td.Input)); err != nil { 256 return fmt.Sprintf("clone: %s", err.Error()) 257 } else if foundAny { 258 cloneOpts.IterOptions = &iterOpts 259 } 260 var err error 261 iters[to], err = fromIter.Clone(cloneOpts) 262 if err != nil { 263 return err.Error() 264 } 265 return "" 266 case "commit": 267 name := pluckStringCmdArg(td, "batch") 268 b := batches[name] 269 defer b.Close() 270 count := b.Count() 271 require.NoError(t, d.Apply(b, nil)) 272 delete(batches, name) 273 return fmt.Sprintf("committed %d keys\n", count) 274 case "combined-iter": 275 o := &IterOptions{KeyTypes: IterKeyTypePointsAndRanges} 276 var reader Reader = d 277 var name string 278 for _, arg := range td.CmdArgs { 279 switch arg.Key { 280 case "mask-suffix": 281 o.RangeKeyMasking.Suffix = []byte(arg.Vals[0]) 282 case "mask-filter": 283 o.RangeKeyMasking.Filter = func() BlockPropertyFilterMask { 284 return sstable.NewTestKeysMaskingFilter() 285 } 286 case "lower": 287 o.LowerBound = []byte(arg.Vals[0]) 288 case "upper": 289 o.UpperBound = []byte(arg.Vals[0]) 290 case "name": 291 name = arg.Vals[0] 292 case "reader": 293 reader = batches[arg.Vals[0]] 294 if reader == nil { 295 return fmt.Sprintf("unknown reader %q", arg.Vals[0]) 296 } 297 case "point-key-filter": 298 if len(arg.Vals) != 2 { 299 return fmt.Sprintf("blockprop-filter expects 2 arguments, received %d", len(arg.Vals)) 300 } 301 min, err := strconv.ParseUint(arg.Vals[0], 10, 64) 302 if err != nil { 303 return err.Error() 304 } 305 max, err := strconv.ParseUint(arg.Vals[1], 10, 64) 306 if err != nil { 307 return err.Error() 308 } 309 o.PointKeyFilters = []sstable.BlockPropertyFilter{ 310 sstable.NewTestKeysBlockPropertyFilter(min, max), 311 } 312 o.SkipPoint = func(k []byte) bool { 313 i := testkeys.Comparer.Split(k) 314 if i == len(k) { 315 return false 316 } 317 v, err := testkeys.ParseSuffix(k[i:]) 318 if err != nil { 319 return false 320 } 321 return uint64(v) < min || uint64(v) >= max 322 } 323 case "snapshot": 324 s, err := strconv.ParseUint(arg.Vals[0], 10, 64) 325 if err != nil { 326 return err.Error() 327 } 328 func() { 329 d.mu.Lock() 330 defer d.mu.Unlock() 331 l := &d.mu.snapshots 332 for i := l.root.next; i != &l.root; i = i.next { 333 if i.seqNum == s { 334 reader = i 335 break 336 } 337 } 338 }() 339 case "use-l6-filter": 340 o.UseL6Filters = true 341 } 342 } 343 var iter *Iterator 344 var err error 345 func() { 346 defer func() { 347 if r := recover(); r != nil { 348 switch v := r.(type) { 349 case string: 350 err = errors.New(v) 351 case error: 352 err = v 353 default: 354 panic(r) 355 } 356 } 357 }() 358 iter = newIter(name, reader, o) 359 }() 360 if err != nil { 361 return err.Error() 362 } 363 return runIterCmd(td, iter, name == "" /* close iter */) 364 case "rangekey-iter": 365 name := pluckStringCmdArg(td, "name") 366 iter := newIter(name, d, &IterOptions{KeyTypes: IterKeyTypeRangesOnly}) 367 return runIterCmd(td, iter, name == "" /* close iter */) 368 case "scan-rangekeys": 369 var buf bytes.Buffer 370 iter := newIter( 371 pluckStringCmdArg(td, "name"), 372 d, 373 &IterOptions{KeyTypes: IterKeyTypeRangesOnly}, 374 ) 375 func() { 376 defer iter.Close() 377 for iter.First(); iter.Valid(); iter.Next() { 378 start, end := iter.RangeBounds() 379 fmt.Fprintf(&buf, "[%s, %s)\n", start, end) 380 writeRangeKeys(&buf, iter) 381 fmt.Fprintln(&buf) 382 } 383 }() 384 return buf.String() 385 case "iter": 386 var name string 387 td.ScanArgs(t, "iter", &name) 388 return runIterCmd(td, iters[name], false /* close iter */) 389 case "wait-table-stats": 390 d.mu.Lock() 391 d.waitTableStats() 392 d.mu.Unlock() 393 return "" 394 default: 395 return fmt.Sprintf("unknown command %q", td.Cmd) 396 } 397 }) 398 }) 399 } 400 401 func pluckStringCmdArg(td *datadriven.TestData, key string) string { 402 if arg, ok := td.Arg(key); ok { 403 return arg.Vals[0] 404 } 405 return "" 406 }