github.com/cockroachdb/pebble@v1.1.1-0.20240513155919-3622ade60459/checkpoint_test.go (about) 1 // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 // of this source code is governed by a BSD-style license that can be found in 3 // the LICENSE file. 4 5 package pebble 6 7 import ( 8 "bytes" 9 "context" 10 "fmt" 11 "math/rand" 12 "sort" 13 "strings" 14 "sync" 15 "testing" 16 17 "github.com/cockroachdb/datadriven" 18 "github.com/cockroachdb/pebble/internal/base" 19 "github.com/cockroachdb/pebble/vfs" 20 "github.com/stretchr/testify/require" 21 ) 22 23 func TestCheckpoint(t *testing.T) { 24 dbs := make(map[string]*DB) 25 defer func() { 26 for _, db := range dbs { 27 if db.closed.Load() == nil { 28 require.NoError(t, db.Close()) 29 } 30 } 31 }() 32 33 mem := vfs.NewMem() 34 var memLog base.InMemLogger 35 opts := &Options{ 36 FS: vfs.WithLogging(mem, memLog.Infof), 37 FormatMajorVersion: internalFormatNewest, 38 L0CompactionThreshold: 10, 39 DisableAutomaticCompactions: true, 40 } 41 opts.private.disableTableStats = true 42 opts.private.testingAlwaysWaitForCleanup = true 43 44 datadriven.RunTest(t, "testdata/checkpoint", func(t *testing.T, td *datadriven.TestData) string { 45 switch td.Cmd { 46 case "batch": 47 if len(td.CmdArgs) != 1 { 48 return "batch <db>" 49 } 50 memLog.Reset() 51 d := dbs[td.CmdArgs[0].String()] 52 b := d.NewBatch() 53 if err := runBatchDefineCmd(td, b); err != nil { 54 return err.Error() 55 } 56 if err := b.Commit(Sync); err != nil { 57 return err.Error() 58 } 59 return memLog.String() 60 61 case "checkpoint": 62 if !(len(td.CmdArgs) == 2 || (len(td.CmdArgs) == 3 && td.CmdArgs[2].Key == "restrict")) { 63 return "checkpoint <db> <dir> [restrict=(start-end, ...)]" 64 } 65 var opts []CheckpointOption 66 if len(td.CmdArgs) == 3 { 67 var spans []CheckpointSpan 68 for _, v := range td.CmdArgs[2].Vals { 69 splits := strings.SplitN(v, "-", 2) 70 if len(splits) != 2 { 71 return fmt.Sprintf("invalid restrict range %q", v) 72 } 73 spans = append(spans, CheckpointSpan{ 74 Start: []byte(splits[0]), 75 End: []byte(splits[1]), 76 }) 77 } 78 opts = append(opts, WithRestrictToSpans(spans)) 79 } 80 memLog.Reset() 81 d := dbs[td.CmdArgs[0].String()] 82 if err := d.Checkpoint(td.CmdArgs[1].String(), opts...); err != nil { 83 return err.Error() 84 } 85 return memLog.String() 86 87 case "ingest-and-excise": 88 d := dbs[td.CmdArgs[0].String()] 89 90 // Hacky but the command doesn't expect a db string. Get rid of it. 91 td.CmdArgs = td.CmdArgs[1:] 92 if err := runIngestAndExciseCmd(td, d, mem); err != nil { 93 return err.Error() 94 } 95 return "" 96 97 case "build": 98 d := dbs[td.CmdArgs[0].String()] 99 100 // Hacky but the command doesn't expect a db string. Get rid of it. 101 td.CmdArgs = td.CmdArgs[1:] 102 if err := runBuildCmd(td, d, mem); err != nil { 103 return err.Error() 104 } 105 return "" 106 107 case "lsm": 108 d := dbs[td.CmdArgs[0].String()] 109 110 // Hacky but the command doesn't expect a db string. Get rid of it. 111 td.CmdArgs = td.CmdArgs[1:] 112 return runLSMCmd(td, d) 113 114 case "compact": 115 if len(td.CmdArgs) != 1 { 116 return "compact <db>" 117 } 118 memLog.Reset() 119 d := dbs[td.CmdArgs[0].String()] 120 if err := d.Compact(nil, []byte("\xff"), false); err != nil { 121 return err.Error() 122 } 123 d.TestOnlyWaitForCleaning() 124 return memLog.String() 125 126 case "print-backing": 127 // prints contents of the file backing map in the version. Used to 128 // test whether the checkpoint removed the filebackings correctly. 129 if len(td.CmdArgs) != 1 { 130 return "print-backing <db>" 131 } 132 d := dbs[td.CmdArgs[0].String()] 133 d.mu.Lock() 134 d.mu.versions.logLock() 135 var fileNums []base.DiskFileNum 136 for _, b := range d.mu.versions.backingState.fileBackingMap { 137 fileNums = append(fileNums, b.DiskFileNum) 138 } 139 d.mu.versions.logUnlock() 140 d.mu.Unlock() 141 142 sort.Slice(fileNums, func(i, j int) bool { 143 return uint64(fileNums[i].FileNum()) < uint64(fileNums[j].FileNum()) 144 }) 145 var buf bytes.Buffer 146 for _, f := range fileNums { 147 buf.WriteString(fmt.Sprintf("%s\n", f.String())) 148 } 149 return buf.String() 150 151 case "close": 152 if len(td.CmdArgs) != 1 { 153 return "close <db>" 154 } 155 d := dbs[td.CmdArgs[0].String()] 156 require.NoError(t, d.Close()) 157 return "" 158 159 case "flush": 160 if len(td.CmdArgs) != 1 { 161 return "flush <db>" 162 } 163 memLog.Reset() 164 d := dbs[td.CmdArgs[0].String()] 165 if err := d.Flush(); err != nil { 166 return err.Error() 167 } 168 return memLog.String() 169 170 case "list": 171 if len(td.CmdArgs) != 1 { 172 return "list <dir>" 173 } 174 paths, err := mem.List(td.CmdArgs[0].String()) 175 if err != nil { 176 return err.Error() 177 } 178 sort.Strings(paths) 179 return fmt.Sprintf("%s\n", strings.Join(paths, "\n")) 180 181 case "open": 182 if len(td.CmdArgs) != 1 && len(td.CmdArgs) != 2 { 183 return "open <dir> [readonly]" 184 } 185 opts.ReadOnly = false 186 if len(td.CmdArgs) == 2 { 187 if td.CmdArgs[1].String() != "readonly" { 188 return "open <dir> [readonly]" 189 } 190 opts.ReadOnly = true 191 } 192 193 memLog.Reset() 194 dir := td.CmdArgs[0].String() 195 d, err := Open(dir, opts) 196 if err != nil { 197 return err.Error() 198 } 199 dbs[dir] = d 200 return memLog.String() 201 202 case "scan": 203 if len(td.CmdArgs) != 1 { 204 return "scan <db>" 205 } 206 memLog.Reset() 207 d := dbs[td.CmdArgs[0].String()] 208 iter, _ := d.NewIter(nil) 209 for valid := iter.First(); valid; valid = iter.Next() { 210 memLog.Infof("%s %s", iter.Key(), iter.Value()) 211 } 212 memLog.Infof(".") 213 if err := iter.Close(); err != nil { 214 memLog.Infof("%v\n", err) 215 } 216 return memLog.String() 217 218 default: 219 return fmt.Sprintf("unknown command: %s", td.Cmd) 220 } 221 }) 222 } 223 224 func TestCheckpointCompaction(t *testing.T) { 225 fs := vfs.NewMem() 226 d, err := Open("", &Options{FS: fs}) 227 require.NoError(t, err) 228 229 ctx, cancel := context.WithCancel(context.Background()) 230 231 var wg sync.WaitGroup 232 wg.Add(4) 233 go func() { 234 defer cancel() 235 defer wg.Done() 236 for i := 0; ctx.Err() == nil; i++ { 237 if err := d.Set([]byte(fmt.Sprintf("key%06d", i)), nil, nil); err != nil { 238 t.Error(err) 239 return 240 } 241 } 242 }() 243 go func() { 244 defer cancel() 245 defer wg.Done() 246 for ctx.Err() == nil { 247 if err := d.Compact([]byte("key"), []byte("key999999"), false); err != nil { 248 t.Error(err) 249 return 250 } 251 } 252 }() 253 check := make(chan string, 100) 254 go func() { 255 defer cancel() 256 defer close(check) 257 defer wg.Done() 258 for i := 0; ctx.Err() == nil && i < 200; i++ { 259 dir := fmt.Sprintf("checkpoint%06d", i) 260 if err := d.Checkpoint(dir); err != nil { 261 t.Error(err) 262 return 263 } 264 select { 265 case <-ctx.Done(): 266 return 267 case check <- dir: 268 } 269 } 270 }() 271 go func() { 272 opts := &Options{FS: fs} 273 defer cancel() 274 defer wg.Done() 275 for dir := range check { 276 d2, err := Open(dir, opts) 277 if err != nil { 278 t.Error(err) 279 return 280 } 281 // Check the checkpoint has all the sstables that the manifest 282 // claims it has. 283 tableInfos, _ := d2.SSTables() 284 for _, tables := range tableInfos { 285 for _, tbl := range tables { 286 if tbl.Virtual { 287 continue 288 } 289 if _, err := fs.Stat(base.MakeFilepath(fs, dir, base.FileTypeTable, tbl.FileNum.DiskFileNum())); err != nil { 290 t.Error(err) 291 return 292 } 293 } 294 } 295 if err := d2.Close(); err != nil { 296 t.Error(err) 297 return 298 } 299 } 300 }() 301 <-ctx.Done() 302 wg.Wait() 303 require.NoError(t, d.Close()) 304 } 305 306 func TestCheckpointFlushWAL(t *testing.T) { 307 const checkpointPath = "checkpoints/checkpoint" 308 fs := vfs.NewStrictMem() 309 opts := &Options{FS: fs} 310 key, value := []byte("key"), []byte("value") 311 312 // Create a checkpoint from an unsynced DB. 313 { 314 d, err := Open("", opts) 315 require.NoError(t, err) 316 { 317 wb := d.NewBatch() 318 err = wb.Set(key, value, nil) 319 require.NoError(t, err) 320 err = d.Apply(wb, NoSync) 321 require.NoError(t, err) 322 } 323 err = d.Checkpoint(checkpointPath, WithFlushedWAL()) 324 require.NoError(t, err) 325 require.NoError(t, d.Close()) 326 fs.ResetToSyncedState() 327 } 328 329 // Check that the WAL has been flushed in the checkpoint. 330 { 331 files, err := fs.List(checkpointPath) 332 require.NoError(t, err) 333 hasLogFile := false 334 for _, f := range files { 335 info, err := fs.Stat(fs.PathJoin(checkpointPath, f)) 336 require.NoError(t, err) 337 if strings.HasSuffix(f, ".log") { 338 hasLogFile = true 339 require.NotZero(t, info.Size()) 340 } 341 } 342 require.True(t, hasLogFile) 343 } 344 345 // Check that the checkpoint contains the expected data. 346 { 347 d, err := Open(checkpointPath, opts) 348 require.NoError(t, err) 349 iter, _ := d.NewIter(nil) 350 require.True(t, iter.First()) 351 require.Equal(t, key, iter.Key()) 352 require.Equal(t, value, iter.Value()) 353 require.False(t, iter.Next()) 354 require.NoError(t, iter.Close()) 355 require.NoError(t, d.Close()) 356 } 357 } 358 359 func TestCheckpointManyFiles(t *testing.T) { 360 if testing.Short() { 361 t.Skip("skipping because of short flag") 362 } 363 const checkpointPath = "checkpoint" 364 opts := &Options{ 365 FS: vfs.NewMem(), 366 FormatMajorVersion: internalFormatNewest, 367 DisableAutomaticCompactions: true, 368 } 369 // Disable compression to speed up the test. 370 opts.EnsureDefaults() 371 for i := range opts.Levels { 372 opts.Levels[i].Compression = NoCompression 373 } 374 375 d, err := Open("", opts) 376 require.NoError(t, err) 377 defer d.Close() 378 379 mkKey := func(x int) []byte { 380 return []byte(fmt.Sprintf("key%06d", x)) 381 } 382 // We want to test the case where the appended record with the excluded files 383 // makes the manifest cross 32KB. This will happen for a range of values 384 // around 450. 385 n := 400 + rand.Intn(100) 386 for i := 0; i < n; i++ { 387 err := d.Set(mkKey(i), nil, nil) 388 require.NoError(t, err) 389 err = d.Flush() 390 require.NoError(t, err) 391 } 392 err = d.Checkpoint(checkpointPath, WithRestrictToSpans([]CheckpointSpan{ 393 { 394 Start: mkKey(0), 395 End: mkKey(10), 396 }, 397 })) 398 require.NoError(t, err) 399 400 // Open the checkpoint and iterate through all the keys. 401 { 402 d, err := Open(checkpointPath, opts) 403 require.NoError(t, err) 404 iter, _ := d.NewIter(nil) 405 require.True(t, iter.First()) 406 require.NoError(t, iter.Error()) 407 n := 1 408 for iter.Next() { 409 n++ 410 } 411 require.NoError(t, iter.Error()) 412 require.NoError(t, iter.Close()) 413 require.NoError(t, d.Close()) 414 require.Equal(t, 10, n) 415 } 416 }