github.com/cockroachdb/pebble@v1.1.2/metamorphic/test.go (about) 1 // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 // of this source code is governed by a BSD-style license that can be found in 3 // the LICENSE file. 4 5 package metamorphic 6 7 import ( 8 "fmt" 9 "io" 10 "os" 11 "sort" 12 "strings" 13 14 "github.com/cockroachdb/errors" 15 "github.com/cockroachdb/pebble" 16 "github.com/cockroachdb/pebble/vfs" 17 "github.com/cockroachdb/pebble/vfs/errorfs" 18 ) 19 20 type test struct { 21 // The list of ops to execute. The ops refer to slots in the batches, iters, 22 // and snapshots slices. 23 ops []op 24 opsWaitOn [][]int // op index -> op indexes 25 opsDone []chan struct{} // op index -> done channel 26 idx int 27 // The DB the test is run on. 28 dir string 29 db *pebble.DB 30 opts *pebble.Options 31 testOpts *TestOptions 32 writeOpts *pebble.WriteOptions 33 tmpDir string 34 // The slots for the batches, iterators, and snapshots. These are read and 35 // written by the ops to pass state from one op to another. 36 batches []*pebble.Batch 37 iters []*retryableIter 38 snapshots []readerCloser 39 } 40 41 func newTest(ops []op) *test { 42 return &test{ 43 ops: ops, 44 } 45 } 46 47 func (t *test) init(h *history, dir string, testOpts *TestOptions) error { 48 t.dir = dir 49 t.testOpts = testOpts 50 t.writeOpts = pebble.NoSync 51 if testOpts.strictFS { 52 t.writeOpts = pebble.Sync 53 } 54 t.opts = testOpts.Opts.EnsureDefaults() 55 t.opts.Logger = h 56 lel := pebble.MakeLoggingEventListener(t.opts.Logger) 57 t.opts.EventListener = &lel 58 t.opts.DebugCheck = func(db *pebble.DB) error { 59 // Wrap the ordinary DebugCheckLevels with retrying 60 // of injected errors. 61 return withRetries(func() error { 62 return pebble.DebugCheckLevels(db) 63 }) 64 } 65 66 t.opsWaitOn, t.opsDone = computeSynchronizationPoints(t.ops) 67 68 defer t.opts.Cache.Unref() 69 70 // If an error occurs and we were using an in-memory FS, attempt to clone to 71 // on-disk in order to allow post-mortem debugging. Note that always using 72 // the on-disk FS isn't desirable because there is a large performance 73 // difference between in-memory and on-disk which causes different code paths 74 // and timings to be exercised. 75 maybeExit := func(err error) { 76 if err == nil || errors.Is(err, errorfs.ErrInjected) || errors.Is(err, pebble.ErrCancelledCompaction) { 77 return 78 } 79 t.maybeSaveData() 80 fmt.Fprintln(os.Stderr, err) 81 os.Exit(1) 82 } 83 84 // Exit early on any error from a background operation. 85 t.opts.EventListener.BackgroundError = func(err error) { 86 t.opts.Logger.Infof("background error: %s", err) 87 maybeExit(err) 88 } 89 t.opts.EventListener.CompactionEnd = func(info pebble.CompactionInfo) { 90 t.opts.Logger.Infof("%s", info) 91 maybeExit(info.Err) 92 } 93 t.opts.EventListener.FlushEnd = func(info pebble.FlushInfo) { 94 t.opts.Logger.Infof("%s", info) 95 if info.Err != nil && !strings.Contains(info.Err.Error(), "pebble: empty table") { 96 maybeExit(info.Err) 97 } 98 } 99 t.opts.EventListener.ManifestCreated = func(info pebble.ManifestCreateInfo) { 100 t.opts.Logger.Infof("%s", info) 101 maybeExit(info.Err) 102 } 103 t.opts.EventListener.ManifestDeleted = func(info pebble.ManifestDeleteInfo) { 104 t.opts.Logger.Infof("%s", info) 105 maybeExit(info.Err) 106 } 107 t.opts.EventListener.TableDeleted = func(info pebble.TableDeleteInfo) { 108 t.opts.Logger.Infof("%s", info) 109 maybeExit(info.Err) 110 } 111 t.opts.EventListener.TableIngested = func(info pebble.TableIngestInfo) { 112 t.opts.Logger.Infof("%s", info) 113 maybeExit(info.Err) 114 } 115 t.opts.EventListener.WALCreated = func(info pebble.WALCreateInfo) { 116 t.opts.Logger.Infof("%s", info) 117 maybeExit(info.Err) 118 } 119 t.opts.EventListener.WALDeleted = func(info pebble.WALDeleteInfo) { 120 t.opts.Logger.Infof("%s", info) 121 maybeExit(info.Err) 122 } 123 124 for i := range t.testOpts.CustomOpts { 125 if err := t.testOpts.CustomOpts[i].Open(t.opts); err != nil { 126 return err 127 } 128 } 129 130 var db *pebble.DB 131 var err error 132 err = withRetries(func() error { 133 db, err = pebble.Open(dir, t.opts) 134 return err 135 }) 136 if err != nil { 137 return err 138 } 139 h.log.Printf("// db.Open() %v", err) 140 141 if t.testOpts.sharedStorageEnabled { 142 err = withRetries(func() error { 143 return db.SetCreatorID(1) 144 }) 145 if err != nil { 146 return err 147 } 148 h.log.Printf("// db.SetCreatorID() %v", err) 149 } 150 151 t.tmpDir = t.opts.FS.PathJoin(dir, "tmp") 152 if err = t.opts.FS.MkdirAll(t.tmpDir, 0755); err != nil { 153 return err 154 } 155 if t.testOpts.strictFS { 156 // Sync the whole directory path for the tmpDir, since restartDB() is executed during 157 // the test. That would reset MemFS to the synced state, which would make an unsynced 158 // directory disappear in the middle of the test. It is the responsibility of the test 159 // (not Pebble) to ensure that it can write the ssts that it will subsequently ingest 160 // into Pebble. 161 for { 162 f, err := t.opts.FS.OpenDir(dir) 163 if err != nil { 164 return err 165 } 166 if err = f.Sync(); err != nil { 167 return err 168 } 169 if err = f.Close(); err != nil { 170 return err 171 } 172 if len(dir) == 1 { 173 break 174 } 175 dir = t.opts.FS.PathDir(dir) 176 // TODO(sbhola): PathDir returns ".", which OpenDir() complains about. Fix. 177 if len(dir) == 1 { 178 dir = "/" 179 } 180 } 181 } 182 183 t.db = db 184 return nil 185 } 186 187 func (t *test) isFMV(fmv pebble.FormatMajorVersion) bool { 188 return t.db.FormatMajorVersion() >= fmv 189 } 190 191 func (t *test) restartDB() error { 192 if !t.testOpts.strictFS { 193 return nil 194 } 195 t.opts.Cache.Ref() 196 // The fs isn't necessarily a MemFS. 197 fs, ok := vfs.Root(t.opts.FS).(*vfs.MemFS) 198 if ok { 199 fs.SetIgnoreSyncs(true) 200 } 201 if err := t.db.Close(); err != nil { 202 return err 203 } 204 // Release any resources held by custom options. This may be used, for 205 // example, by the encryption-at-rest custom option (within the Cockroach 206 // repository) to close the file registry. 207 for i := range t.testOpts.CustomOpts { 208 if err := t.testOpts.CustomOpts[i].Close(t.opts); err != nil { 209 return err 210 } 211 } 212 if ok { 213 fs.ResetToSyncedState() 214 fs.SetIgnoreSyncs(false) 215 } 216 217 // TODO(jackson): Audit errorRate and ensure custom options' hooks semantics 218 // are well defined within the context of retries. 219 err := withRetries(func() (err error) { 220 // Reacquire any resources required by custom options. This may be used, for 221 // example, by the encryption-at-rest custom option (within the Cockroach 222 // repository) to reopen the file registry. 223 for i := range t.testOpts.CustomOpts { 224 if err := t.testOpts.CustomOpts[i].Open(t.opts); err != nil { 225 return err 226 } 227 } 228 t.db, err = pebble.Open(t.dir, t.opts) 229 return err 230 }) 231 t.opts.Cache.Unref() 232 return err 233 } 234 235 // If an in-memory FS is being used, save the contents to disk. 236 func (t *test) maybeSaveData() { 237 rootFS := vfs.Root(t.opts.FS) 238 if rootFS == vfs.Default { 239 return 240 } 241 _ = os.RemoveAll(t.dir) 242 if _, err := vfs.Clone(rootFS, vfs.Default, t.dir, t.dir); err != nil { 243 t.opts.Logger.Infof("unable to clone: %s: %v", t.dir, err) 244 } 245 } 246 247 func (t *test) step(h *history) bool { 248 if t.idx >= len(t.ops) { 249 return false 250 } 251 t.ops[t.idx].run(t, h.recorder(-1 /* thread */, t.idx)) 252 t.idx++ 253 return true 254 } 255 256 func (t *test) setBatch(id objID, b *pebble.Batch) { 257 if id.tag() != batchTag { 258 panic(fmt.Sprintf("invalid batch ID: %s", id)) 259 } 260 t.batches[id.slot()] = b 261 } 262 263 func (t *test) setIter(id objID, i *pebble.Iterator) { 264 if id.tag() != iterTag { 265 panic(fmt.Sprintf("invalid iter ID: %s", id)) 266 } 267 t.iters[id.slot()] = &retryableIter{ 268 iter: i, 269 lastKey: nil, 270 } 271 } 272 273 type readerCloser interface { 274 pebble.Reader 275 io.Closer 276 } 277 278 func (t *test) setSnapshot(id objID, s readerCloser) { 279 if id.tag() != snapTag { 280 panic(fmt.Sprintf("invalid snapshot ID: %s", id)) 281 } 282 t.snapshots[id.slot()] = s 283 } 284 285 func (t *test) clearObj(id objID) { 286 switch id.tag() { 287 case dbTag: 288 t.db = nil 289 case batchTag: 290 t.batches[id.slot()] = nil 291 case iterTag: 292 t.iters[id.slot()] = nil 293 case snapTag: 294 t.snapshots[id.slot()] = nil 295 } 296 } 297 298 func (t *test) getBatch(id objID) *pebble.Batch { 299 if id.tag() != batchTag { 300 panic(fmt.Sprintf("invalid batch ID: %s", id)) 301 } 302 return t.batches[id.slot()] 303 } 304 305 func (t *test) getCloser(id objID) io.Closer { 306 switch id.tag() { 307 case dbTag: 308 return t.db 309 case batchTag: 310 return t.batches[id.slot()] 311 case iterTag: 312 return t.iters[id.slot()] 313 case snapTag: 314 return t.snapshots[id.slot()] 315 } 316 panic(fmt.Sprintf("cannot close ID: %s", id)) 317 } 318 319 func (t *test) getIter(id objID) *retryableIter { 320 if id.tag() != iterTag { 321 panic(fmt.Sprintf("invalid iter ID: %s", id)) 322 } 323 return t.iters[id.slot()] 324 } 325 326 func (t *test) getReader(id objID) pebble.Reader { 327 switch id.tag() { 328 case dbTag: 329 return t.db 330 case batchTag: 331 return t.batches[id.slot()] 332 case snapTag: 333 return t.snapshots[id.slot()] 334 } 335 panic(fmt.Sprintf("invalid reader ID: %s", id)) 336 } 337 338 func (t *test) getWriter(id objID) pebble.Writer { 339 switch id.tag() { 340 case dbTag: 341 return t.db 342 case batchTag: 343 return t.batches[id.slot()] 344 } 345 panic(fmt.Sprintf("invalid writer ID: %s", id)) 346 } 347 348 // Compute the synchronization points between operations. When operating 349 // with more than 1 thread, operations must synchronize access to shared 350 // objects. Compute two slices the same length as ops. 351 // 352 // opsWaitOn: the value v at index i indicates that operation i must wait 353 // for the operation at index v to finish before it may run. NB: v < i 354 // 355 // opsDone: the channel at index i must be closed when the operation at index i 356 // completes. This slice is sparse. Operations that are never used as 357 // synchronization points may have a nil channel. 358 func computeSynchronizationPoints(ops []op) (opsWaitOn [][]int, opsDone []chan struct{}) { 359 opsDone = make([]chan struct{}, len(ops)) // operation index -> done channel 360 opsWaitOn = make([][]int, len(ops)) // operation index -> operation index 361 lastOpReference := make(map[objID]int) // objID -> operation index 362 for i, o := range ops { 363 // Find the last operation that involved the same receiver object. We at 364 // least need to wait on that operation. 365 receiver := o.receiver() 366 waitIndex, ok := lastOpReference[receiver] 367 lastOpReference[receiver] = i 368 if !ok { 369 // Only valid for i=0. For all other operations, the receiver should 370 // have been referenced by some other operation before it's used as 371 // a receiver. 372 if i != 0 { 373 panic(fmt.Sprintf("op %d on receiver %s; first reference of %s", i, receiver, receiver)) 374 } 375 continue 376 } 377 378 // The last operation that referenced `receiver` is the one at index 379 // `waitIndex`. All operations with the same receiver are performed on 380 // the same thread. We only need to synchronize on the operation at 381 // `waitIndex` if `receiver` isn't also the receiver on that operation 382 // too. 383 if ops[waitIndex].receiver() != receiver { 384 opsWaitOn[i] = append(opsWaitOn[i], waitIndex) 385 } 386 387 // In additional to synchronizing on the operation's receiver operation, 388 // we may need to synchronize on additional objects. For example, 389 // batch0.Commit() must synchronize its receiver, batch0, but also on 390 // dbObjID since it mutates database state. 391 for _, syncObjID := range o.syncObjs() { 392 if vi, vok := lastOpReference[syncObjID]; vok { 393 opsWaitOn[i] = append(opsWaitOn[i], vi) 394 } 395 lastOpReference[syncObjID] = i 396 } 397 398 waitIndexes := opsWaitOn[i] 399 sort.Ints(waitIndexes) 400 for _, waitIndex := range waitIndexes { 401 // If this is the first operation that must wait on the operation at 402 // `waitIndex`, then there will be no channel for the operation yet. 403 // Create one. 404 if opsDone[waitIndex] == nil { 405 opsDone[waitIndex] = make(chan struct{}) 406 } 407 } 408 } 409 return opsWaitOn, opsDone 410 }