github.com/cockroachdb/pebble@v1.1.1-0.20240513155919-3622ade60459/compaction_picker_test.go (about) 1 // Copyright 2018 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 // of this source code is governed by a BSD-style license that can be found in 3 // the LICENSE file. 4 5 package pebble 6 7 import ( 8 "bytes" 9 "fmt" 10 "math" 11 "sort" 12 "strconv" 13 "strings" 14 "sync" 15 "testing" 16 "time" 17 18 "github.com/cockroachdb/datadriven" 19 "github.com/cockroachdb/errors" 20 "github.com/cockroachdb/pebble/internal/base" 21 "github.com/cockroachdb/pebble/internal/humanize" 22 "github.com/cockroachdb/pebble/internal/manifest" 23 "github.com/cockroachdb/pebble/internal/testkeys" 24 "github.com/cockroachdb/pebble/vfs" 25 "github.com/stretchr/testify/require" 26 ) 27 28 func loadVersion(t *testing.T, d *datadriven.TestData) (*version, *Options, string) { 29 var sizes [numLevels]int64 30 opts := &Options{} 31 opts.testingRandomized(t) 32 opts.EnsureDefaults() 33 34 if len(d.CmdArgs) != 1 { 35 return nil, nil, fmt.Sprintf("%s expects 1 argument", d.Cmd) 36 } 37 var err error 38 opts.LBaseMaxBytes, err = strconv.ParseInt(d.CmdArgs[0].Key, 10, 64) 39 if err != nil { 40 return nil, nil, err.Error() 41 } 42 43 var files [numLevels][]*fileMetadata 44 if len(d.Input) > 0 { 45 // Parse each line as 46 // 47 // <level>: <size> [compensation] 48 // 49 // Creating sstables within the level whose file sizes total to `size` 50 // and whose compensated file sizes total to `size`+`compensation`. If 51 // size is sufficiently large, only one single file is created. See 52 // the TODO below. 53 for _, data := range strings.Split(d.Input, "\n") { 54 parts := strings.Split(data, " ") 55 parts[0] = strings.TrimSuffix(strings.TrimSpace(parts[0]), ":") 56 if len(parts) < 2 { 57 return nil, nil, fmt.Sprintf("malformed test:\n%s", d.Input) 58 } 59 level, err := strconv.Atoi(parts[0]) 60 if err != nil { 61 return nil, nil, err.Error() 62 } 63 if files[level] != nil { 64 return nil, nil, fmt.Sprintf("level %d already filled", level) 65 } 66 size, err := strconv.ParseUint(strings.TrimSpace(parts[1]), 10, 64) 67 if err != nil { 68 return nil, nil, err.Error() 69 } 70 var compensation uint64 71 if len(parts) == 3 { 72 compensation, err = strconv.ParseUint(strings.TrimSpace(parts[2]), 10, 64) 73 if err != nil { 74 return nil, nil, err.Error() 75 } 76 } 77 78 var lastFile *fileMetadata 79 for i := uint64(1); sizes[level] < int64(size); i++ { 80 var key InternalKey 81 if level == 0 { 82 // For L0, make `size` overlapping files. 83 key = base.MakeInternalKey([]byte(fmt.Sprintf("%04d", 1)), i, InternalKeyKindSet) 84 } else { 85 key = base.MakeInternalKey([]byte(fmt.Sprintf("%04d", i)), i, InternalKeyKindSet) 86 } 87 m := (&fileMetadata{ 88 FileNum: base.FileNum(uint64(level)*100_000 + i), 89 SmallestSeqNum: key.SeqNum(), 90 LargestSeqNum: key.SeqNum(), 91 Size: 1, 92 Stats: manifest.TableStats{ 93 RangeDeletionsBytesEstimate: 0, 94 }, 95 }).ExtendPointKeyBounds(opts.Comparer.Compare, key, key) 96 m.InitPhysicalBacking() 97 m.StatsMarkValid() 98 lastFile = m 99 if size >= 100 { 100 // If the requested size of the level is very large only add a single 101 // file in order to avoid massive blow-up in the number of files in 102 // the Version. 103 // 104 // TODO(peter): There is tension between the testing in 105 // TestCompactionPickerLevelMaxBytes and 106 // TestCompactionPickerTargetLevel. Clean this up somehow. 107 m.Size = size 108 if level != 0 { 109 endKey := base.MakeInternalKey([]byte(fmt.Sprintf("%04d", size)), i, InternalKeyKindSet) 110 m.ExtendPointKeyBounds(opts.Comparer.Compare, key, endKey) 111 } 112 } 113 files[level] = append(files[level], m) 114 sizes[level] += int64(m.Size) 115 } 116 // Let all the compensation be due to the last file. 117 if lastFile != nil && compensation > 0 { 118 lastFile.Stats.RangeDeletionsBytesEstimate = compensation 119 } 120 } 121 } 122 123 vers := newVersion(opts, files) 124 return vers, opts, "" 125 } 126 127 func TestCompactionPickerByScoreLevelMaxBytes(t *testing.T) { 128 datadriven.RunTest(t, "testdata/compaction_picker_level_max_bytes", 129 func(t *testing.T, d *datadriven.TestData) string { 130 switch d.Cmd { 131 case "init": 132 vers, opts, errMsg := loadVersion(t, d) 133 if errMsg != "" { 134 return errMsg 135 } 136 137 p, ok := newCompactionPicker(vers, opts, nil).(*compactionPickerByScore) 138 require.True(t, ok) 139 var buf bytes.Buffer 140 for level := p.getBaseLevel(); level < numLevels; level++ { 141 fmt.Fprintf(&buf, "%d: %d\n", level, p.levelMaxBytes[level]) 142 } 143 return buf.String() 144 145 default: 146 return fmt.Sprintf("unknown command: %s", d.Cmd) 147 } 148 }) 149 } 150 151 func TestCompactionPickerTargetLevel(t *testing.T) { 152 var vers *version 153 var opts *Options 154 var pickerByScore *compactionPickerByScore 155 156 parseInProgress := func(vals []string) ([]compactionInfo, error) { 157 var levels []int 158 for _, s := range vals { 159 l, err := strconv.ParseInt(s, 10, 8) 160 if err != nil { 161 return nil, err 162 } 163 levels = append(levels, int(l)) 164 } 165 if len(levels)%2 != 0 { 166 return nil, errors.New("odd number of levels with ongoing compactions") 167 } 168 var inProgress []compactionInfo 169 for i := 0; i < len(levels); i += 2 { 170 inProgress = append(inProgress, compactionInfo{ 171 inputs: []compactionLevel{ 172 {level: levels[i]}, 173 {level: levels[i+1]}, 174 }, 175 outputLevel: levels[i+1], 176 }) 177 } 178 return inProgress, nil 179 } 180 181 resetCompacting := func() { 182 for _, files := range vers.Levels { 183 files.Slice().Each(func(f *fileMetadata) { 184 f.CompactionState = manifest.CompactionStateNotCompacting 185 }) 186 } 187 } 188 189 datadriven.RunTest(t, "testdata/compaction_picker_target_level", 190 func(t *testing.T, d *datadriven.TestData) string { 191 switch d.Cmd { 192 case "init": 193 // loadVersion expects a single datadriven argument that it 194 // sets as Options.LBaseMaxBytes. It parses the input as 195 // newline-separated levels, specifying the level's file size 196 // and optionally additional compensation to be added during 197 // compensated file size calculations. Eg: 198 // 199 // init <LBaseMaxBytes> 200 // <level>: <size> [compensation] 201 // <level>: <size> [compensation] 202 var errMsg string 203 vers, opts, errMsg = loadVersion(t, d) 204 if errMsg != "" { 205 return errMsg 206 } 207 return runVersionFileSizes(vers) 208 case "init_cp": 209 resetCompacting() 210 211 var inProgress []compactionInfo 212 if arg, ok := d.Arg("ongoing"); ok { 213 var err error 214 inProgress, err = parseInProgress(arg.Vals) 215 if err != nil { 216 return err.Error() 217 } 218 } 219 220 p := newCompactionPicker(vers, opts, inProgress) 221 var ok bool 222 pickerByScore, ok = p.(*compactionPickerByScore) 223 require.True(t, ok) 224 return fmt.Sprintf("base: %d", pickerByScore.baseLevel) 225 case "queue": 226 var b strings.Builder 227 var inProgress []compactionInfo 228 for { 229 env := compactionEnv{ 230 diskAvailBytes: math.MaxUint64, 231 earliestUnflushedSeqNum: InternalKeySeqNumMax, 232 inProgressCompactions: inProgress, 233 } 234 pc := pickerByScore.pickAuto(env) 235 if pc == nil { 236 break 237 } 238 fmt.Fprintf(&b, "L%d->L%d: %.1f\n", pc.startLevel.level, pc.outputLevel.level, pc.score) 239 inProgress = append(inProgress, compactionInfo{ 240 inputs: pc.inputs, 241 outputLevel: pc.outputLevel.level, 242 smallest: pc.smallest, 243 largest: pc.largest, 244 }) 245 if pc.outputLevel.level == 0 { 246 // Once we pick one L0->L0 compaction, we'll keep on doing so 247 // because the test isn't marking files as Compacting. 248 break 249 } 250 for _, cl := range pc.inputs { 251 cl.files.Each(func(f *fileMetadata) { 252 f.CompactionState = manifest.CompactionStateCompacting 253 fmt.Fprintf(&b, " %s marked as compacting\n", f) 254 }) 255 } 256 } 257 258 resetCompacting() 259 return b.String() 260 case "pick": 261 resetCompacting() 262 263 var inProgress []compactionInfo 264 if len(d.CmdArgs) == 1 { 265 arg := d.CmdArgs[0] 266 if arg.Key != "ongoing" { 267 return "unknown arg: " + arg.Key 268 } 269 var err error 270 inProgress, err = parseInProgress(arg.Vals) 271 if err != nil { 272 return err.Error() 273 } 274 } 275 276 // Mark files as compacting for each in-progress compaction. 277 for i := range inProgress { 278 c := &inProgress[i] 279 for j, cl := range c.inputs { 280 iter := vers.Levels[cl.level].Iter() 281 for f := iter.First(); f != nil; f = iter.Next() { 282 if !f.IsCompacting() { 283 f.CompactionState = manifest.CompactionStateCompacting 284 c.inputs[j].files = iter.Take().Slice() 285 break 286 } 287 } 288 } 289 if c.inputs[0].level == 0 && c.outputLevel != 0 { 290 // L0->Lbase: mark all of Lbase as compacting. 291 c.inputs[1].files = vers.Levels[c.outputLevel].Slice() 292 for _, in := range c.inputs { 293 in.files.Each(func(f *fileMetadata) { 294 f.CompactionState = manifest.CompactionStateCompacting 295 }) 296 } 297 } 298 } 299 300 var b strings.Builder 301 fmt.Fprintf(&b, "Initial state before pick:\n%s", runVersionFileSizes(vers)) 302 pc := pickerByScore.pickAuto(compactionEnv{ 303 earliestUnflushedSeqNum: InternalKeySeqNumMax, 304 inProgressCompactions: inProgress, 305 }) 306 if pc != nil { 307 fmt.Fprintf(&b, "Picked: L%d->L%d: %0.1f\n", pc.startLevel.level, pc.outputLevel.level, pc.score) 308 } 309 if pc == nil { 310 fmt.Fprintln(&b, "Picked: no compaction") 311 } 312 return b.String() 313 case "pick_manual": 314 var startLevel int 315 var start, end string 316 d.MaybeScanArgs(t, "level", &startLevel) 317 d.MaybeScanArgs(t, "start", &start) 318 d.MaybeScanArgs(t, "end", &end) 319 320 iStart := base.MakeInternalKey([]byte(start), InternalKeySeqNumMax, InternalKeyKindMax) 321 iEnd := base.MakeInternalKey([]byte(end), 0, 0) 322 manual := &manualCompaction{ 323 done: make(chan error, 1), 324 level: startLevel, 325 start: iStart.UserKey, 326 end: iEnd.UserKey, 327 } 328 329 pc, retryLater := pickManualCompaction( 330 pickerByScore.vers, 331 opts, 332 compactionEnv{ 333 earliestUnflushedSeqNum: InternalKeySeqNumMax, 334 }, 335 pickerByScore.getBaseLevel(), 336 manual) 337 if pc == nil { 338 return fmt.Sprintf("nil, retryLater = %v", retryLater) 339 } 340 341 return fmt.Sprintf("L%d->L%d, retryLater = %v", pc.startLevel.level, pc.outputLevel.level, retryLater) 342 default: 343 return fmt.Sprintf("unknown command: %s", d.Cmd) 344 } 345 }) 346 } 347 348 func TestCompactionPickerEstimatedCompactionDebt(t *testing.T) { 349 datadriven.RunTest(t, "testdata/compaction_picker_estimated_debt", 350 func(t *testing.T, d *datadriven.TestData) string { 351 switch d.Cmd { 352 case "init": 353 vers, opts, errMsg := loadVersion(t, d) 354 if errMsg != "" { 355 return errMsg 356 } 357 opts.MemTableSize = 1000 358 359 p := newCompactionPicker(vers, opts, nil) 360 return fmt.Sprintf("%d\n", p.estimatedCompactionDebt(0)) 361 362 default: 363 return fmt.Sprintf("unknown command: %s", d.Cmd) 364 } 365 }) 366 } 367 368 func TestCompactionPickerL0(t *testing.T) { 369 opts := (*Options)(nil).EnsureDefaults() 370 opts.Experimental.L0CompactionConcurrency = 1 371 372 parseMeta := func(s string) (*fileMetadata, error) { 373 parts := strings.Split(s, ":") 374 fileNum, err := strconv.Atoi(parts[0]) 375 if err != nil { 376 return nil, err 377 } 378 fields := strings.Fields(parts[1]) 379 parts = strings.Split(fields[0], "-") 380 if len(parts) != 2 { 381 return nil, errors.Errorf("malformed table spec: %s", s) 382 } 383 m := (&fileMetadata{ 384 FileNum: base.FileNum(fileNum), 385 }).ExtendPointKeyBounds( 386 opts.Comparer.Compare, 387 base.ParseInternalKey(strings.TrimSpace(parts[0])), 388 base.ParseInternalKey(strings.TrimSpace(parts[1])), 389 ) 390 m.SmallestSeqNum = m.Smallest.SeqNum() 391 m.LargestSeqNum = m.Largest.SeqNum() 392 m.InitPhysicalBacking() 393 return m, nil 394 } 395 396 var picker *compactionPickerByScore 397 var inProgressCompactions []compactionInfo 398 var pc *pickedCompaction 399 400 datadriven.RunTest(t, "testdata/compaction_picker_L0", func(t *testing.T, td *datadriven.TestData) string { 401 switch td.Cmd { 402 case "define": 403 fileMetas := [manifest.NumLevels][]*fileMetadata{} 404 baseLevel := manifest.NumLevels - 1 405 level := 0 406 var err error 407 lines := strings.Split(td.Input, "\n") 408 var compactionLines []string 409 410 for len(lines) > 0 { 411 data := strings.TrimSpace(lines[0]) 412 lines = lines[1:] 413 switch data { 414 case "L0", "L1", "L2", "L3", "L4", "L5", "L6": 415 level, err = strconv.Atoi(data[1:]) 416 if err != nil { 417 return err.Error() 418 } 419 case "compactions": 420 compactionLines, lines = lines, nil 421 default: 422 meta, err := parseMeta(data) 423 if err != nil { 424 return err.Error() 425 } 426 if level != 0 && level < baseLevel { 427 baseLevel = level 428 } 429 fileMetas[level] = append(fileMetas[level], meta) 430 } 431 } 432 433 // Parse in-progress compactions in the form of: 434 // L0 000001 -> L2 000005 435 inProgressCompactions = nil 436 for len(compactionLines) > 0 { 437 parts := strings.Fields(compactionLines[0]) 438 compactionLines = compactionLines[1:] 439 440 var level int 441 var info compactionInfo 442 first := true 443 compactionFiles := map[int][]*fileMetadata{} 444 for _, p := range parts { 445 switch p { 446 case "L0", "L1", "L2", "L3", "L4", "L5", "L6": 447 var err error 448 level, err = strconv.Atoi(p[1:]) 449 if err != nil { 450 return err.Error() 451 } 452 if len(info.inputs) > 0 && info.inputs[len(info.inputs)-1].level == level { 453 // eg, L0 -> L0 compaction or L6 -> L6 compaction 454 continue 455 } 456 if info.outputLevel < level { 457 info.outputLevel = level 458 } 459 info.inputs = append(info.inputs, compactionLevel{level: level}) 460 case "->": 461 continue 462 default: 463 fileNum, err := strconv.Atoi(p) 464 if err != nil { 465 return err.Error() 466 } 467 var compactFile *fileMetadata 468 for _, m := range fileMetas[level] { 469 if m.FileNum == FileNum(fileNum) { 470 compactFile = m 471 } 472 } 473 if compactFile == nil { 474 return fmt.Sprintf("cannot find compaction file %s", FileNum(fileNum)) 475 } 476 compactFile.CompactionState = manifest.CompactionStateCompacting 477 if first || base.InternalCompare(DefaultComparer.Compare, info.largest, compactFile.Largest) < 0 { 478 info.largest = compactFile.Largest 479 } 480 if first || base.InternalCompare(DefaultComparer.Compare, info.smallest, compactFile.Smallest) > 0 { 481 info.smallest = compactFile.Smallest 482 } 483 first = false 484 compactionFiles[level] = append(compactionFiles[level], compactFile) 485 } 486 } 487 for i, cl := range info.inputs { 488 files := compactionFiles[cl.level] 489 info.inputs[i].files = manifest.NewLevelSliceSeqSorted(files) 490 // Mark as intra-L0 compacting if the compaction is 491 // L0 -> L0. 492 if info.outputLevel == 0 { 493 for _, f := range files { 494 f.IsIntraL0Compacting = true 495 } 496 } 497 } 498 inProgressCompactions = append(inProgressCompactions, info) 499 } 500 501 version := newVersion(opts, fileMetas) 502 version.L0Sublevels.InitCompactingFileInfo(inProgressL0Compactions(inProgressCompactions)) 503 vs := &versionSet{ 504 opts: opts, 505 cmp: DefaultComparer.Compare, 506 cmpName: DefaultComparer.Name, 507 } 508 vs.versions.Init(nil) 509 vs.append(version) 510 picker = &compactionPickerByScore{ 511 opts: opts, 512 vers: version, 513 baseLevel: baseLevel, 514 } 515 vs.picker = picker 516 picker.initLevelMaxBytes(inProgressCompactions) 517 518 var buf bytes.Buffer 519 fmt.Fprint(&buf, version.String()) 520 if len(inProgressCompactions) > 0 { 521 fmt.Fprintln(&buf, "compactions") 522 for _, c := range inProgressCompactions { 523 fmt.Fprintf(&buf, " %s\n", c.String()) 524 } 525 } 526 return buf.String() 527 case "pick-auto": 528 td.MaybeScanArgs(t, "l0_compaction_threshold", &opts.L0CompactionThreshold) 529 td.MaybeScanArgs(t, "l0_compaction_file_threshold", &opts.L0CompactionFileThreshold) 530 531 pc = picker.pickAuto(compactionEnv{ 532 diskAvailBytes: math.MaxUint64, 533 earliestUnflushedSeqNum: math.MaxUint64, 534 inProgressCompactions: inProgressCompactions, 535 }) 536 var result strings.Builder 537 if pc != nil { 538 checkClone(t, pc) 539 c := newCompaction(pc, opts, time.Now(), nil /* provider */) 540 fmt.Fprintf(&result, "L%d -> L%d\n", pc.startLevel.level, pc.outputLevel.level) 541 fmt.Fprintf(&result, "L%d: %s\n", pc.startLevel.level, fileNums(pc.startLevel.files)) 542 if !pc.outputLevel.files.Empty() { 543 fmt.Fprintf(&result, "L%d: %s\n", pc.outputLevel.level, fileNums(pc.outputLevel.files)) 544 } 545 if !c.grandparents.Empty() { 546 fmt.Fprintf(&result, "grandparents: %s\n", fileNums(c.grandparents)) 547 } 548 } else { 549 return "nil" 550 } 551 return result.String() 552 case "mark-for-compaction": 553 var fileNum uint64 554 td.ScanArgs(t, "file", &fileNum) 555 for l, lm := range picker.vers.Levels { 556 iter := lm.Iter() 557 for f := iter.First(); f != nil; f = iter.Next() { 558 if f.FileNum != base.FileNum(fileNum) { 559 continue 560 } 561 f.MarkedForCompaction = true 562 picker.vers.Stats.MarkedForCompaction++ 563 picker.vers.Levels[l].InvalidateAnnotation(markedForCompactionAnnotator{}) 564 return fmt.Sprintf("marked L%d.%s", l, f.FileNum) 565 } 566 } 567 return "not-found" 568 case "max-output-file-size": 569 if pc == nil { 570 return "no compaction" 571 } 572 return fmt.Sprintf("%d", pc.maxOutputFileSize) 573 case "max-overlap-bytes": 574 if pc == nil { 575 return "no compaction" 576 } 577 return fmt.Sprintf("%d", pc.maxOverlapBytes) 578 } 579 return fmt.Sprintf("unrecognized command: %s", td.Cmd) 580 }) 581 } 582 583 func TestCompactionPickerConcurrency(t *testing.T) { 584 opts := (*Options)(nil).EnsureDefaults() 585 opts.Experimental.L0CompactionConcurrency = 1 586 587 parseMeta := func(s string) (*fileMetadata, error) { 588 parts := strings.Split(s, ":") 589 fileNum, err := strconv.Atoi(parts[0]) 590 if err != nil { 591 return nil, err 592 } 593 fields := strings.Fields(parts[1]) 594 parts = strings.Split(fields[0], "-") 595 if len(parts) != 2 { 596 return nil, errors.Errorf("malformed table spec: %s", s) 597 } 598 m := (&fileMetadata{ 599 FileNum: base.FileNum(fileNum), 600 Size: 1028, 601 }).ExtendPointKeyBounds( 602 opts.Comparer.Compare, 603 base.ParseInternalKey(strings.TrimSpace(parts[0])), 604 base.ParseInternalKey(strings.TrimSpace(parts[1])), 605 ) 606 m.InitPhysicalBacking() 607 for _, p := range fields[1:] { 608 if strings.HasPrefix(p, "size=") { 609 v, err := strconv.Atoi(strings.TrimPrefix(p, "size=")) 610 if err != nil { 611 return nil, err 612 } 613 m.Size = uint64(v) 614 } 615 } 616 m.SmallestSeqNum = m.Smallest.SeqNum() 617 m.LargestSeqNum = m.Largest.SeqNum() 618 return m, nil 619 } 620 621 var picker *compactionPickerByScore 622 var inProgressCompactions []compactionInfo 623 624 datadriven.RunTest(t, "testdata/compaction_picker_concurrency", func(t *testing.T, td *datadriven.TestData) string { 625 switch td.Cmd { 626 case "define": 627 fileMetas := [manifest.NumLevels][]*fileMetadata{} 628 level := 0 629 var err error 630 lines := strings.Split(td.Input, "\n") 631 var compactionLines []string 632 633 for len(lines) > 0 { 634 data := strings.TrimSpace(lines[0]) 635 lines = lines[1:] 636 switch data { 637 case "L0", "L1", "L2", "L3", "L4", "L5", "L6": 638 level, err = strconv.Atoi(data[1:]) 639 if err != nil { 640 return err.Error() 641 } 642 case "compactions": 643 compactionLines, lines = lines, nil 644 default: 645 meta, err := parseMeta(data) 646 if err != nil { 647 return err.Error() 648 } 649 fileMetas[level] = append(fileMetas[level], meta) 650 } 651 } 652 653 // Parse in-progress compactions in the form of: 654 // L0 000001 -> L2 000005 655 inProgressCompactions = nil 656 for len(compactionLines) > 0 { 657 parts := strings.Fields(compactionLines[0]) 658 compactionLines = compactionLines[1:] 659 660 var level int 661 var info compactionInfo 662 first := true 663 compactionFiles := map[int][]*fileMetadata{} 664 for _, p := range parts { 665 switch p { 666 case "L0", "L1", "L2", "L3", "L4", "L5", "L6": 667 var err error 668 level, err = strconv.Atoi(p[1:]) 669 if err != nil { 670 return err.Error() 671 } 672 if len(info.inputs) > 0 && info.inputs[len(info.inputs)-1].level == level { 673 // eg, L0 -> L0 compaction or L6 -> L6 compaction 674 continue 675 } 676 if info.outputLevel < level { 677 info.outputLevel = level 678 } 679 info.inputs = append(info.inputs, compactionLevel{level: level}) 680 case "->": 681 continue 682 default: 683 fileNum, err := strconv.Atoi(p) 684 if err != nil { 685 return err.Error() 686 } 687 var compactFile *fileMetadata 688 for _, m := range fileMetas[level] { 689 if m.FileNum == FileNum(fileNum) { 690 compactFile = m 691 } 692 } 693 if compactFile == nil { 694 return fmt.Sprintf("cannot find compaction file %s", FileNum(fileNum)) 695 } 696 compactFile.CompactionState = manifest.CompactionStateCompacting 697 if first || base.InternalCompare(DefaultComparer.Compare, info.largest, compactFile.Largest) < 0 { 698 info.largest = compactFile.Largest 699 } 700 if first || base.InternalCompare(DefaultComparer.Compare, info.smallest, compactFile.Smallest) > 0 { 701 info.smallest = compactFile.Smallest 702 } 703 first = false 704 compactionFiles[level] = append(compactionFiles[level], compactFile) 705 } 706 } 707 for i, cl := range info.inputs { 708 files := compactionFiles[cl.level] 709 if cl.level == 0 { 710 info.inputs[i].files = manifest.NewLevelSliceSeqSorted(files) 711 } else { 712 info.inputs[i].files = manifest.NewLevelSliceKeySorted(DefaultComparer.Compare, files) 713 } 714 // Mark as intra-L0 compacting if the compaction is 715 // L0 -> L0. 716 if info.outputLevel == 0 { 717 for _, f := range files { 718 f.IsIntraL0Compacting = true 719 } 720 } 721 } 722 inProgressCompactions = append(inProgressCompactions, info) 723 } 724 725 version := newVersion(opts, fileMetas) 726 version.L0Sublevels.InitCompactingFileInfo(inProgressL0Compactions(inProgressCompactions)) 727 vs := &versionSet{ 728 opts: opts, 729 cmp: DefaultComparer.Compare, 730 cmpName: DefaultComparer.Name, 731 } 732 vs.versions.Init(nil) 733 vs.append(version) 734 735 picker = newCompactionPicker(version, opts, inProgressCompactions).(*compactionPickerByScore) 736 vs.picker = picker 737 738 var buf bytes.Buffer 739 fmt.Fprint(&buf, version.String()) 740 if len(inProgressCompactions) > 0 { 741 fmt.Fprintln(&buf, "compactions") 742 for _, c := range inProgressCompactions { 743 fmt.Fprintf(&buf, " %s\n", c.String()) 744 } 745 } 746 return buf.String() 747 748 case "pick-auto": 749 td.MaybeScanArgs(t, "l0_compaction_threshold", &opts.L0CompactionThreshold) 750 td.MaybeScanArgs(t, "l0_compaction_concurrency", &opts.Experimental.L0CompactionConcurrency) 751 td.MaybeScanArgs(t, "compaction_debt_concurrency", &opts.Experimental.CompactionDebtConcurrency) 752 753 pc := picker.pickAuto(compactionEnv{ 754 earliestUnflushedSeqNum: math.MaxUint64, 755 inProgressCompactions: inProgressCompactions, 756 }) 757 var result strings.Builder 758 if pc != nil { 759 c := newCompaction(pc, opts, time.Now(), nil /* provider */) 760 fmt.Fprintf(&result, "L%d -> L%d\n", pc.startLevel.level, pc.outputLevel.level) 761 fmt.Fprintf(&result, "L%d: %s\n", pc.startLevel.level, fileNums(pc.startLevel.files)) 762 if !pc.outputLevel.files.Empty() { 763 fmt.Fprintf(&result, "L%d: %s\n", pc.outputLevel.level, fileNums(pc.outputLevel.files)) 764 } 765 if !c.grandparents.Empty() { 766 fmt.Fprintf(&result, "grandparents: %s\n", fileNums(c.grandparents)) 767 } 768 } else { 769 return "nil" 770 } 771 return result.String() 772 } 773 return fmt.Sprintf("unrecognized command: %s", td.Cmd) 774 }) 775 } 776 777 func TestCompactionPickerPickReadTriggered(t *testing.T) { 778 opts := (*Options)(nil).EnsureDefaults() 779 var picker *compactionPickerByScore 780 var rcList readCompactionQueue 781 var vers *version 782 783 parseMeta := func(s string) (*fileMetadata, error) { 784 parts := strings.Split(s, ":") 785 fileNum, err := strconv.Atoi(parts[0]) 786 if err != nil { 787 return nil, err 788 } 789 fields := strings.Fields(parts[1]) 790 parts = strings.Split(fields[0], "-") 791 if len(parts) != 2 { 792 return nil, errors.Errorf("malformed table spec: %s. usage: <file-num>:start.SET.1-end.SET.2", s) 793 } 794 m := (&fileMetadata{ 795 FileNum: base.FileNum(fileNum), 796 Size: 1028, 797 }).ExtendPointKeyBounds( 798 opts.Comparer.Compare, 799 base.ParseInternalKey(strings.TrimSpace(parts[0])), 800 base.ParseInternalKey(strings.TrimSpace(parts[1])), 801 ) 802 m.InitPhysicalBacking() 803 for _, p := range fields[1:] { 804 if strings.HasPrefix(p, "size=") { 805 v, err := strconv.Atoi(strings.TrimPrefix(p, "size=")) 806 if err != nil { 807 return nil, err 808 } 809 m.Size = uint64(v) 810 } 811 } 812 m.SmallestSeqNum = m.Smallest.SeqNum() 813 m.LargestSeqNum = m.Largest.SeqNum() 814 return m, nil 815 } 816 817 datadriven.RunTest(t, "testdata/compaction_picker_read_triggered", func(t *testing.T, td *datadriven.TestData) string { 818 switch td.Cmd { 819 case "define": 820 rcList = readCompactionQueue{} 821 fileMetas := [manifest.NumLevels][]*fileMetadata{} 822 level := 0 823 var err error 824 lines := strings.Split(td.Input, "\n") 825 826 for len(lines) > 0 { 827 data := strings.TrimSpace(lines[0]) 828 lines = lines[1:] 829 switch data { 830 case "L0", "L1", "L2", "L3", "L4", "L5", "L6": 831 level, err = strconv.Atoi(data[1:]) 832 if err != nil { 833 return err.Error() 834 } 835 default: 836 meta, err := parseMeta(data) 837 if err != nil { 838 return err.Error() 839 } 840 fileMetas[level] = append(fileMetas[level], meta) 841 } 842 } 843 844 vers = newVersion(opts, fileMetas) 845 vs := &versionSet{ 846 opts: opts, 847 cmp: DefaultComparer.Compare, 848 cmpName: DefaultComparer.Name, 849 } 850 vs.versions.Init(nil) 851 vs.append(vers) 852 var inProgressCompactions []compactionInfo 853 picker = newCompactionPicker(vers, opts, inProgressCompactions).(*compactionPickerByScore) 854 vs.picker = picker 855 856 var buf bytes.Buffer 857 fmt.Fprint(&buf, vers.String()) 858 return buf.String() 859 860 case "add-read-compaction": 861 for _, line := range strings.Split(td.Input, "\n") { 862 if line == "" { 863 continue 864 } 865 parts := strings.Split(line, " ") 866 if len(parts) != 3 { 867 return "error: malformed data for add-read-compaction. usage: <level>: <start>-<end> <filenum>" 868 } 869 if l, err := strconv.Atoi(parts[0][:1]); err == nil { 870 keys := strings.Split(parts[1], "-") 871 fileNum, _ := strconv.Atoi(parts[2]) 872 873 rc := readCompaction{ 874 level: l, 875 start: []byte(keys[0]), 876 end: []byte(keys[1]), 877 fileNum: base.FileNum(fileNum), 878 } 879 rcList.add(&rc, DefaultComparer.Compare) 880 } else { 881 return err.Error() 882 } 883 } 884 return "" 885 886 case "show-read-compactions": 887 var sb strings.Builder 888 if rcList.size == 0 { 889 sb.WriteString("(none)") 890 } 891 for i := 0; i < rcList.size; i++ { 892 rc := rcList.at(i) 893 sb.WriteString(fmt.Sprintf("(level: %d, start: %s, end: %s)\n", rc.level, string(rc.start), string(rc.end))) 894 } 895 return sb.String() 896 897 case "pick-auto": 898 pc := picker.pickAuto(compactionEnv{ 899 earliestUnflushedSeqNum: math.MaxUint64, 900 readCompactionEnv: readCompactionEnv{ 901 readCompactions: &rcList, 902 flushing: false, 903 }, 904 }) 905 var result strings.Builder 906 if pc != nil { 907 fmt.Fprintf(&result, "L%d -> L%d\n", pc.startLevel.level, pc.outputLevel.level) 908 fmt.Fprintf(&result, "L%d: %s\n", pc.startLevel.level, fileNums(pc.startLevel.files)) 909 if !pc.outputLevel.files.Empty() { 910 fmt.Fprintf(&result, "L%d: %s\n", pc.outputLevel.level, fileNums(pc.outputLevel.files)) 911 } 912 } else { 913 return "nil" 914 } 915 return result.String() 916 } 917 return fmt.Sprintf("unrecognized command: %s", td.Cmd) 918 }) 919 } 920 921 type alwaysMultiLevel struct{} 922 923 func (d alwaysMultiLevel) pick( 924 pcOrig *pickedCompaction, opts *Options, diskAvailBytes uint64, 925 ) *pickedCompaction { 926 pcMulti := pcOrig.clone() 927 if !pcMulti.setupMultiLevelCandidate(opts, diskAvailBytes) { 928 return pcOrig 929 } 930 return pcMulti 931 } 932 933 func (d alwaysMultiLevel) allowL0() bool { 934 return false 935 } 936 937 func TestPickedCompactionSetupInputs(t *testing.T) { 938 opts := &Options{} 939 opts.EnsureDefaults() 940 941 parseMeta := func(s string) *fileMetadata { 942 parts := strings.Split(strings.TrimSpace(s), " ") 943 var fileSize uint64 944 var compacting bool 945 for _, part := range parts { 946 switch { 947 case part == "compacting": 948 compacting = true 949 case strings.HasPrefix(part, "size="): 950 v, err := strconv.ParseUint(strings.TrimPrefix(part, "size="), 10, 64) 951 require.NoError(t, err) 952 fileSize = v 953 } 954 } 955 tableParts := strings.Split(parts[0], "-") 956 if len(tableParts) != 2 { 957 t.Fatalf("malformed table spec: %s", s) 958 } 959 state := manifest.CompactionStateNotCompacting 960 if compacting { 961 state = manifest.CompactionStateCompacting 962 } 963 m := (&fileMetadata{ 964 CompactionState: state, 965 Size: fileSize, 966 }).ExtendPointKeyBounds( 967 opts.Comparer.Compare, 968 base.ParseInternalKey(strings.TrimSpace(tableParts[0])), 969 base.ParseInternalKey(strings.TrimSpace(tableParts[1])), 970 ) 971 m.SmallestSeqNum = m.Smallest.SeqNum() 972 m.LargestSeqNum = m.Largest.SeqNum() 973 m.InitPhysicalBacking() 974 return m 975 } 976 977 setupInputTest := func(t *testing.T, d *datadriven.TestData) string { 978 switch d.Cmd { 979 case "setup-inputs": 980 var availBytes uint64 = math.MaxUint64 981 var maxLevelBytes [7]int64 982 args := d.CmdArgs 983 984 if len(args) > 0 && args[0].Key == "avail-bytes" { 985 require.Equal(t, 1, len(args[0].Vals)) 986 var err error 987 availBytes, err = strconv.ParseUint(args[0].Vals[0], 10, 64) 988 require.NoError(t, err) 989 args = args[1:] 990 } 991 992 if len(args) != 2 { 993 return "setup-inputs [avail-bytes=XXX] <start> <end>" 994 } 995 996 pc := &pickedCompaction{ 997 cmp: DefaultComparer.Compare, 998 inputs: []compactionLevel{{level: -1}, {level: -1}}, 999 } 1000 pc.startLevel, pc.outputLevel = &pc.inputs[0], &pc.inputs[1] 1001 var currentLevel int 1002 var files [numLevels][]*fileMetadata 1003 fileNum := FileNum(1) 1004 1005 for _, data := range strings.Split(d.Input, "\n") { 1006 switch data[:2] { 1007 case "L0", "L1", "L2", "L3", "L4", "L5", "L6": 1008 levelArgs := strings.Fields(data) 1009 level, err := strconv.Atoi(levelArgs[0][1:]) 1010 if err != nil { 1011 return err.Error() 1012 } 1013 currentLevel = level 1014 if len(levelArgs) > 1 { 1015 maxSizeArg := strings.Replace(levelArgs[1], "max-size=", "", 1) 1016 maxSize, err := strconv.ParseInt(maxSizeArg, 10, 64) 1017 if err != nil { 1018 return err.Error() 1019 } 1020 maxLevelBytes[level] = maxSize 1021 } else { 1022 maxLevelBytes[level] = math.MaxInt64 1023 } 1024 if pc.startLevel.level == -1 { 1025 pc.startLevel.level = level 1026 1027 } else if pc.outputLevel.level == -1 { 1028 if pc.startLevel.level >= level { 1029 return fmt.Sprintf("startLevel=%d >= outputLevel=%d\n", pc.startLevel.level, level) 1030 } 1031 pc.outputLevel.level = level 1032 } 1033 default: 1034 meta := parseMeta(data) 1035 meta.FileNum = fileNum 1036 fileNum++ 1037 files[currentLevel] = append(files[currentLevel], meta) 1038 } 1039 } 1040 1041 if pc.outputLevel.level == -1 { 1042 pc.outputLevel.level = pc.startLevel.level + 1 1043 } 1044 pc.version = newVersion(opts, files) 1045 pc.startLevel.files = pc.version.Overlaps(pc.startLevel.level, pc.cmp, 1046 []byte(args[0].String()), []byte(args[1].String()), false /* exclusiveEnd */) 1047 1048 var isCompacting bool 1049 if !pc.setupInputs(opts, availBytes, pc.startLevel) { 1050 isCompacting = true 1051 } 1052 origPC := pc 1053 pc = pc.maybeAddLevel(opts, availBytes) 1054 // If pc points to a new pickedCompaction, a new multi level compaction 1055 // was initialized. 1056 initMultiLevel := pc != origPC 1057 checkClone(t, pc) 1058 var buf bytes.Buffer 1059 for _, cl := range pc.inputs { 1060 if cl.files.Empty() { 1061 continue 1062 } 1063 1064 fmt.Fprintf(&buf, "L%d\n", cl.level) 1065 cl.files.Each(func(f *fileMetadata) { 1066 fmt.Fprintf(&buf, " %s\n", f) 1067 }) 1068 } 1069 if isCompacting { 1070 fmt.Fprintf(&buf, "is-compacting\n") 1071 } 1072 1073 if initMultiLevel { 1074 extraLevel := pc.extraLevels[0].level 1075 fmt.Fprintf(&buf, "init-multi-level(%d,%d,%d)\n", pc.startLevel.level, extraLevel, 1076 pc.outputLevel.level) 1077 fmt.Fprintf(&buf, "Original WriteAmp %.2f; ML WriteAmp %.2f\n", origPC.predictedWriteAmp(), pc.predictedWriteAmp()) 1078 fmt.Fprintf(&buf, "Original OverlappingRatio %.2f; ML OverlappingRatio %.2f\n", origPC.overlappingRatio(), pc.overlappingRatio()) 1079 } 1080 return buf.String() 1081 1082 default: 1083 return fmt.Sprintf("unknown command: %s", d.Cmd) 1084 } 1085 } 1086 1087 t.Logf("Test basic setup inputs behavior without multi level compactions") 1088 opts.Experimental.MultiLevelCompactionHeuristic = NoMultiLevel{} 1089 datadriven.RunTest(t, "testdata/compaction_setup_inputs", 1090 setupInputTest) 1091 1092 t.Logf("Turning multi level compaction on") 1093 opts.Experimental.MultiLevelCompactionHeuristic = alwaysMultiLevel{} 1094 datadriven.RunTest(t, "testdata/compaction_setup_inputs_multilevel_dummy", 1095 setupInputTest) 1096 1097 t.Logf("Try Write-Amp Heuristic") 1098 opts.Experimental.MultiLevelCompactionHeuristic = WriteAmpHeuristic{} 1099 datadriven.RunTest(t, "testdata/compaction_setup_inputs_multilevel_write_amp", 1100 setupInputTest) 1101 } 1102 1103 func TestPickedCompactionExpandInputs(t *testing.T) { 1104 opts := &Options{} 1105 opts.EnsureDefaults() 1106 cmp := DefaultComparer.Compare 1107 var files []*fileMetadata 1108 1109 parseMeta := func(s string) *fileMetadata { 1110 parts := strings.Split(s, "-") 1111 if len(parts) != 2 { 1112 t.Fatalf("malformed table spec: %s", s) 1113 } 1114 m := (&fileMetadata{}).ExtendPointKeyBounds( 1115 opts.Comparer.Compare, 1116 base.ParseInternalKey(parts[0]), 1117 base.ParseInternalKey(parts[1]), 1118 ) 1119 m.InitPhysicalBacking() 1120 return m 1121 } 1122 1123 datadriven.RunTest(t, "testdata/compaction_expand_inputs", 1124 func(t *testing.T, d *datadriven.TestData) string { 1125 switch d.Cmd { 1126 case "define": 1127 files = nil 1128 if len(d.Input) == 0 { 1129 return "" 1130 } 1131 for _, data := range strings.Split(d.Input, "\n") { 1132 meta := parseMeta(data) 1133 meta.FileNum = FileNum(len(files)) 1134 files = append(files, meta) 1135 } 1136 manifest.SortBySmallest(files, cmp) 1137 return "" 1138 1139 case "expand-inputs": 1140 pc := &pickedCompaction{ 1141 cmp: cmp, 1142 inputs: []compactionLevel{{level: 1}}, 1143 } 1144 pc.startLevel = &pc.inputs[0] 1145 1146 var filesLevelled [numLevels][]*fileMetadata 1147 filesLevelled[pc.startLevel.level] = files 1148 pc.version = newVersion(opts, filesLevelled) 1149 1150 if len(d.CmdArgs) != 1 { 1151 return fmt.Sprintf("%s expects 1 argument", d.Cmd) 1152 } 1153 index, err := strconv.ParseInt(d.CmdArgs[0].String(), 10, 64) 1154 if err != nil { 1155 return err.Error() 1156 } 1157 1158 // Advance the iterator to position `index`. 1159 iter := pc.version.Levels[pc.startLevel.level].Iter() 1160 _ = iter.First() 1161 for i := int64(0); i < index; i++ { 1162 _ = iter.Next() 1163 } 1164 1165 inputs, _ := expandToAtomicUnit(cmp, iter.Take().Slice(), true /* disableIsCompacting */) 1166 1167 var buf bytes.Buffer 1168 inputs.Each(func(f *fileMetadata) { 1169 fmt.Fprintf(&buf, "%d: %s-%s\n", f.FileNum, f.Smallest, f.Largest) 1170 }) 1171 return buf.String() 1172 1173 default: 1174 return fmt.Sprintf("unknown command: %s", d.Cmd) 1175 } 1176 }) 1177 } 1178 1179 func TestCompactionOutputFileSize(t *testing.T) { 1180 opts := (*Options)(nil).EnsureDefaults() 1181 var picker *compactionPickerByScore 1182 var vers *version 1183 1184 parseMeta := func(s string) (*fileMetadata, error) { 1185 parts := strings.Split(s, ":") 1186 fileNum, err := strconv.Atoi(parts[0]) 1187 if err != nil { 1188 return nil, err 1189 } 1190 fields := strings.Fields(parts[1]) 1191 parts = strings.Split(fields[0], "-") 1192 if len(parts) != 2 { 1193 return nil, errors.Errorf("malformed table spec: %s. usage: <file-num>:start.SET.1-end.SET.2", s) 1194 } 1195 m := (&fileMetadata{ 1196 FileNum: base.FileNum(fileNum), 1197 Size: 1028, 1198 }).ExtendPointKeyBounds( 1199 opts.Comparer.Compare, 1200 base.ParseInternalKey(strings.TrimSpace(parts[0])), 1201 base.ParseInternalKey(strings.TrimSpace(parts[1])), 1202 ) 1203 m.InitPhysicalBacking() 1204 for _, p := range fields[1:] { 1205 if strings.HasPrefix(p, "size=") { 1206 v, err := strconv.Atoi(strings.TrimPrefix(p, "size=")) 1207 if err != nil { 1208 return nil, err 1209 } 1210 m.Size = uint64(v) 1211 } 1212 if strings.HasPrefix(p, "range-deletions-bytes-estimate=") { 1213 v, err := strconv.Atoi(strings.TrimPrefix(p, "range-deletions-bytes-estimate=")) 1214 if err != nil { 1215 return nil, err 1216 } 1217 m.Stats.RangeDeletionsBytesEstimate = uint64(v) 1218 m.Stats.NumDeletions = 1 // At least one range del responsible for the deletion bytes. 1219 m.StatsMarkValid() 1220 } 1221 } 1222 m.SmallestSeqNum = m.Smallest.SeqNum() 1223 m.LargestSeqNum = m.Largest.SeqNum() 1224 return m, nil 1225 } 1226 1227 datadriven.RunTest(t, "testdata/compaction_output_file_size", func(t *testing.T, td *datadriven.TestData) string { 1228 switch td.Cmd { 1229 case "define": 1230 fileMetas := [manifest.NumLevels][]*fileMetadata{} 1231 level := 0 1232 var err error 1233 lines := strings.Split(td.Input, "\n") 1234 1235 for len(lines) > 0 { 1236 data := strings.TrimSpace(lines[0]) 1237 lines = lines[1:] 1238 switch data { 1239 case "L0", "L1", "L2", "L3", "L4", "L5", "L6": 1240 level, err = strconv.Atoi(data[1:]) 1241 if err != nil { 1242 return err.Error() 1243 } 1244 default: 1245 meta, err := parseMeta(data) 1246 if err != nil { 1247 return err.Error() 1248 } 1249 fileMetas[level] = append(fileMetas[level], meta) 1250 } 1251 } 1252 1253 vers = newVersion(opts, fileMetas) 1254 vs := &versionSet{ 1255 opts: opts, 1256 cmp: DefaultComparer.Compare, 1257 cmpName: DefaultComparer.Name, 1258 } 1259 vs.versions.Init(nil) 1260 vs.append(vers) 1261 var inProgressCompactions []compactionInfo 1262 picker = newCompactionPicker(vers, opts, inProgressCompactions).(*compactionPickerByScore) 1263 vs.picker = picker 1264 1265 var buf bytes.Buffer 1266 fmt.Fprint(&buf, vers.String()) 1267 return buf.String() 1268 1269 case "pick-auto": 1270 pc := picker.pickAuto(compactionEnv{ 1271 earliestUnflushedSeqNum: math.MaxUint64, 1272 earliestSnapshotSeqNum: math.MaxUint64, 1273 }) 1274 var buf bytes.Buffer 1275 if pc != nil { 1276 fmt.Fprintf(&buf, "L%d -> L%d\n", pc.startLevel.level, pc.outputLevel.level) 1277 fmt.Fprintf(&buf, "L%d: %s\n", pc.startLevel.level, fileNums(pc.startLevel.files)) 1278 fmt.Fprintf(&buf, "maxOutputFileSize: %d\n", pc.maxOutputFileSize) 1279 } else { 1280 return "nil" 1281 } 1282 return buf.String() 1283 1284 default: 1285 return fmt.Sprintf("unrecognized command: %s", td.Cmd) 1286 } 1287 }) 1288 } 1289 1290 func TestCompactionPickerCompensatedSize(t *testing.T) { 1291 testCases := []struct { 1292 size uint64 1293 pointDelEstimateBytes uint64 1294 rangeDelEstimateBytes uint64 1295 wantBytes uint64 1296 }{ 1297 { 1298 size: 100, 1299 pointDelEstimateBytes: 0, 1300 rangeDelEstimateBytes: 0, 1301 wantBytes: 100, 1302 }, 1303 { 1304 size: 100, 1305 pointDelEstimateBytes: 10, 1306 rangeDelEstimateBytes: 0, 1307 wantBytes: 100 + 10, 1308 }, 1309 { 1310 size: 100, 1311 pointDelEstimateBytes: 10, 1312 rangeDelEstimateBytes: 5, 1313 wantBytes: 100 + 10 + 5, 1314 }, 1315 } 1316 1317 for _, tc := range testCases { 1318 t.Run("", func(t *testing.T) { 1319 f := &fileMetadata{Size: tc.size} 1320 f.InitPhysicalBacking() 1321 f.Stats.PointDeletionsBytesEstimate = tc.pointDelEstimateBytes 1322 f.Stats.RangeDeletionsBytesEstimate = tc.rangeDelEstimateBytes 1323 gotBytes := compensatedSize(f) 1324 require.Equal(t, tc.wantBytes, gotBytes) 1325 }) 1326 } 1327 } 1328 1329 func TestCompactionPickerPickFile(t *testing.T) { 1330 fs := vfs.NewMem() 1331 opts := &Options{ 1332 Comparer: testkeys.Comparer, 1333 FormatMajorVersion: FormatNewest, 1334 FS: fs, 1335 } 1336 1337 d, err := Open("", opts) 1338 require.NoError(t, err) 1339 defer func() { 1340 if d != nil { 1341 require.NoError(t, d.Close()) 1342 } 1343 }() 1344 1345 datadriven.RunTest(t, "testdata/compaction_picker_pick_file", func(t *testing.T, td *datadriven.TestData) string { 1346 switch td.Cmd { 1347 case "define": 1348 require.NoError(t, d.Close()) 1349 1350 d, err = runDBDefineCmd(td, opts) 1351 if err != nil { 1352 return err.Error() 1353 } 1354 d.mu.Lock() 1355 s := d.mu.versions.currentVersion().String() 1356 d.mu.Unlock() 1357 return s 1358 1359 case "file-sizes": 1360 return runTableFileSizesCmd(td, d) 1361 1362 case "pick-file": 1363 s := strings.TrimPrefix(td.CmdArgs[0].String(), "L") 1364 level, err := strconv.Atoi(s) 1365 if err != nil { 1366 return fmt.Sprintf("unable to parse arg %q as level", td.CmdArgs[0].String()) 1367 } 1368 if level == 0 { 1369 panic("L0 picking unimplemented") 1370 } 1371 d.mu.Lock() 1372 defer d.mu.Unlock() 1373 1374 // Use maybeScheduleCompactionPicker to take care of all of the 1375 // initialization of the compaction-picking environment, but never 1376 // pick a compaction; just call pickFile using the user-provided 1377 // level. 1378 var lf manifest.LevelFile 1379 var ok bool 1380 d.maybeScheduleCompactionPicker(func(untypedPicker compactionPicker, env compactionEnv) *pickedCompaction { 1381 p := untypedPicker.(*compactionPickerByScore) 1382 lf, ok = pickCompactionSeedFile(p.vers, opts, level, level+1, env.earliestSnapshotSeqNum) 1383 return nil 1384 }) 1385 if !ok { 1386 return "(none)" 1387 } 1388 return lf.FileMetadata.String() 1389 1390 default: 1391 return fmt.Sprintf("unknown command: %s", td.Cmd) 1392 } 1393 }) 1394 } 1395 1396 type pausableCleaner struct { 1397 mu sync.Mutex 1398 cond sync.Cond 1399 paused bool 1400 cleaner Cleaner 1401 } 1402 1403 func (c *pausableCleaner) Clean(fs vfs.FS, fileType base.FileType, path string) error { 1404 c.mu.Lock() 1405 defer c.mu.Unlock() 1406 for c.paused { 1407 c.cond.Wait() 1408 } 1409 return c.cleaner.Clean(fs, fileType, path) 1410 } 1411 1412 func (c *pausableCleaner) pause() { 1413 c.mu.Lock() 1414 defer c.mu.Unlock() 1415 c.paused = true 1416 } 1417 1418 func (c *pausableCleaner) resume() { 1419 c.mu.Lock() 1420 defer c.mu.Unlock() 1421 c.paused = false 1422 c.cond.Broadcast() 1423 } 1424 1425 func TestCompactionPickerScores(t *testing.T) { 1426 fs := vfs.NewMem() 1427 cleaner := pausableCleaner{cleaner: DeleteCleaner{}} 1428 cleaner.cond.L = &cleaner.mu 1429 opts := &Options{ 1430 Cleaner: &cleaner, 1431 Comparer: testkeys.Comparer, 1432 DisableAutomaticCompactions: true, 1433 FormatMajorVersion: FormatNewest, 1434 FS: fs, 1435 } 1436 1437 d, err := Open("", opts) 1438 require.NoError(t, err) 1439 defer func() { 1440 if d != nil { 1441 cleaner.resume() 1442 require.NoError(t, closeAllSnapshots(d)) 1443 require.NoError(t, d.Close()) 1444 } 1445 }() 1446 1447 var buf bytes.Buffer 1448 datadriven.RunTest(t, "testdata/compaction_picker_scores", func(t *testing.T, td *datadriven.TestData) string { 1449 switch td.Cmd { 1450 case "define": 1451 require.NoError(t, closeAllSnapshots(d)) 1452 require.NoError(t, d.Close()) 1453 1454 if td.HasArg("pause-cleaning") { 1455 cleaner.pause() 1456 } 1457 1458 d, err = runDBDefineCmd(td, opts) 1459 if err != nil { 1460 return err.Error() 1461 } 1462 d.mu.Lock() 1463 s := d.mu.versions.currentVersion().String() 1464 d.mu.Unlock() 1465 return s 1466 1467 case "disable-table-stats": 1468 d.mu.Lock() 1469 d.opts.private.disableTableStats = true 1470 d.mu.Unlock() 1471 return "" 1472 1473 case "enable-table-stats": 1474 d.mu.Lock() 1475 d.opts.private.disableTableStats = false 1476 d.maybeCollectTableStatsLocked() 1477 d.mu.Unlock() 1478 return "" 1479 1480 case "resume-cleaning": 1481 cleaner.resume() 1482 return "" 1483 1484 case "ingest": 1485 if err = runBuildCmd(td, d, d.opts.FS); err != nil { 1486 return err.Error() 1487 } 1488 if err = runIngestCmd(td, d, d.opts.FS); err != nil { 1489 return err.Error() 1490 } 1491 d.mu.Lock() 1492 s := d.mu.versions.currentVersion().String() 1493 d.mu.Unlock() 1494 return s 1495 1496 case "lsm": 1497 return runLSMCmd(td, d) 1498 1499 case "maybe-compact": 1500 buf.Reset() 1501 d.mu.Lock() 1502 d.opts.DisableAutomaticCompactions = false 1503 d.maybeScheduleCompaction() 1504 fmt.Fprintf(&buf, "%d compactions in progress:", d.mu.compact.compactingCount) 1505 for c := range d.mu.compact.inProgress { 1506 fmt.Fprintf(&buf, "\n%s", c) 1507 } 1508 d.opts.DisableAutomaticCompactions = true 1509 d.mu.Unlock() 1510 return buf.String() 1511 1512 case "scores": 1513 waitFor := "completion" 1514 td.MaybeScanArgs(t, "wait-for-compaction", &waitFor) 1515 1516 // Wait for any running compactions to complete before calculating 1517 // scores. Otherwise, the output of this command is 1518 // nondeterministic. 1519 switch waitFor { 1520 case "completion": 1521 d.mu.Lock() 1522 for d.mu.compact.compactingCount > 0 { 1523 d.mu.compact.cond.Wait() 1524 } 1525 d.mu.Unlock() 1526 case "version-edit": 1527 func() { 1528 for { 1529 d.mu.Lock() 1530 wait := len(d.mu.compact.inProgress) > 0 1531 for c := range d.mu.compact.inProgress { 1532 wait = wait && !c.versionEditApplied 1533 } 1534 d.mu.Unlock() 1535 if !wait { 1536 return 1537 } 1538 // d.mu.compact.cond isn't notified until the compaction 1539 // is removed from inProgress, so we need to just sleep 1540 // and check again soon. 1541 time.Sleep(10 * time.Millisecond) 1542 } 1543 }() 1544 default: 1545 panic(fmt.Sprintf("unrecognized `wait-for-compaction` value: %q", waitFor)) 1546 } 1547 1548 buf.Reset() 1549 fmt.Fprintf(&buf, "L Size Score\n") 1550 for l, lm := range d.Metrics().Levels { 1551 if l < numLevels-1 { 1552 fmt.Fprintf(&buf, "L%-3d\t%-7s%.1f\n", l, humanize.Bytes.Int64(lm.Size), lm.Score) 1553 } else { 1554 fmt.Fprintf(&buf, "L%-3d\t%-7s-\n", l, humanize.Bytes.Int64(lm.Size)) 1555 } 1556 } 1557 return buf.String() 1558 1559 case "wait-pending-table-stats": 1560 return runTableStatsCmd(td, d) 1561 1562 default: 1563 return fmt.Sprintf("unknown command: %s", td.Cmd) 1564 } 1565 }) 1566 } 1567 1568 func fileNums(files manifest.LevelSlice) string { 1569 var ss []string 1570 files.Each(func(f *fileMetadata) { 1571 ss = append(ss, f.FileNum.String()) 1572 }) 1573 sort.Strings(ss) 1574 return strings.Join(ss, ",") 1575 } 1576 1577 func checkClone(t *testing.T, pc *pickedCompaction) { 1578 pcClone := pc.clone() 1579 require.Equal(t, pc.String(), pcClone.String()) 1580 1581 // ensure all input files are in new address 1582 for i := range pc.inputs { 1583 // Len could be zero if setup inputs rejected a level 1584 if pc.inputs[i].files.Len() > 0 { 1585 require.NotEqual(t, &pc.inputs[i], &pcClone.inputs[i]) 1586 } 1587 } 1588 for i := range pc.startLevel.l0SublevelInfo { 1589 if pc.startLevel.l0SublevelInfo[i].Len() > 0 { 1590 require.NotEqual(t, &pc.startLevel.l0SublevelInfo[i], &pcClone.startLevel.l0SublevelInfo[i]) 1591 } 1592 } 1593 }