github.com/insionng/yougam@v0.0.0-20170714101924-2bc18d833463/libraries/syndtr/goleveldb/leveldb/corrupt_test.go (about) 1 // Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com> 2 // All rights reserved. 3 // 4 // Use of this source code is governed by a BSD-style license that can be 5 // found in the LICENSE file. 6 7 package leveldb 8 9 import ( 10 "bytes" 11 "fmt" 12 "io" 13 "math/rand" 14 "testing" 15 16 "github.com/insionng/yougam/libraries/syndtr/goleveldb/leveldb/filter" 17 "github.com/insionng/yougam/libraries/syndtr/goleveldb/leveldb/opt" 18 "github.com/insionng/yougam/libraries/syndtr/goleveldb/leveldb/storage" 19 ) 20 21 const ctValSize = 1000 22 23 type dbCorruptHarness struct { 24 dbHarness 25 } 26 27 func newDbCorruptHarnessWopt(t *testing.T, o *opt.Options) *dbCorruptHarness { 28 h := new(dbCorruptHarness) 29 h.init(t, o) 30 return h 31 } 32 33 func newDbCorruptHarness(t *testing.T) *dbCorruptHarness { 34 return newDbCorruptHarnessWopt(t, &opt.Options{ 35 BlockCacheCapacity: 100, 36 Strict: opt.StrictJournalChecksum, 37 }) 38 } 39 40 func (h *dbCorruptHarness) recover() { 41 p := &h.dbHarness 42 t := p.t 43 44 var err error 45 p.db, err = Recover(h.stor, h.o) 46 if err != nil { 47 t.Fatal("Repair: got error: ", err) 48 } 49 } 50 51 func (h *dbCorruptHarness) build(n int) { 52 p := &h.dbHarness 53 t := p.t 54 db := p.db 55 56 batch := new(Batch) 57 for i := 0; i < n; i++ { 58 batch.Reset() 59 batch.Put(tkey(i), tval(i, ctValSize)) 60 err := db.Write(batch, p.wo) 61 if err != nil { 62 t.Fatal("write error: ", err) 63 } 64 } 65 } 66 67 func (h *dbCorruptHarness) buildShuffled(n int, rnd *rand.Rand) { 68 p := &h.dbHarness 69 t := p.t 70 db := p.db 71 72 batch := new(Batch) 73 for i := range rnd.Perm(n) { 74 batch.Reset() 75 batch.Put(tkey(i), tval(i, ctValSize)) 76 err := db.Write(batch, p.wo) 77 if err != nil { 78 t.Fatal("write error: ", err) 79 } 80 } 81 } 82 83 func (h *dbCorruptHarness) deleteRand(n, max int, rnd *rand.Rand) { 84 p := &h.dbHarness 85 t := p.t 86 db := p.db 87 88 batch := new(Batch) 89 for i := 0; i < n; i++ { 90 batch.Reset() 91 batch.Delete(tkey(rnd.Intn(max))) 92 err := db.Write(batch, p.wo) 93 if err != nil { 94 t.Fatal("write error: ", err) 95 } 96 } 97 } 98 99 func (h *dbCorruptHarness) corrupt(ft storage.FileType, fi, offset, n int) { 100 p := &h.dbHarness 101 t := p.t 102 103 fds, _ := p.stor.List(ft) 104 sortFds(fds) 105 if fi < 0 { 106 fi = len(fds) - 1 107 } 108 if fi >= len(fds) { 109 t.Fatalf("no such file with type %q with index %d", ft, fi) 110 } 111 112 fd := fds[fi] 113 r, err := h.stor.Open(fd) 114 if err != nil { 115 t.Fatal("cannot open file: ", err) 116 } 117 x, err := r.Seek(0, 2) 118 if err != nil { 119 t.Fatal("cannot query file size: ", err) 120 } 121 m := int(x) 122 if _, err := r.Seek(0, 0); err != nil { 123 t.Fatal(err) 124 } 125 126 if offset < 0 { 127 if -offset > m { 128 offset = 0 129 } else { 130 offset = m + offset 131 } 132 } 133 if offset > m { 134 offset = m 135 } 136 if offset+n > m { 137 n = m - offset 138 } 139 140 buf := make([]byte, m) 141 _, err = io.ReadFull(r, buf) 142 if err != nil { 143 t.Fatal("cannot read file: ", err) 144 } 145 r.Close() 146 147 for i := 0; i < n; i++ { 148 buf[offset+i] ^= 0x80 149 } 150 151 err = h.stor.Remove(fd) 152 if err != nil { 153 t.Fatal("cannot remove old file: ", err) 154 } 155 w, err := h.stor.Create(fd) 156 if err != nil { 157 t.Fatal("cannot create new file: ", err) 158 } 159 _, err = w.Write(buf) 160 if err != nil { 161 t.Fatal("cannot write new file: ", err) 162 } 163 w.Close() 164 } 165 166 func (h *dbCorruptHarness) removeAll(ft storage.FileType) { 167 fds, err := h.stor.List(ft) 168 if err != nil { 169 h.t.Fatal("get files: ", err) 170 } 171 for _, fd := range fds { 172 if err := h.stor.Remove(fd); err != nil { 173 h.t.Error("remove file: ", err) 174 } 175 } 176 } 177 178 func (h *dbCorruptHarness) forceRemoveAll(ft storage.FileType) { 179 fds, err := h.stor.List(ft) 180 if err != nil { 181 h.t.Fatal("get files: ", err) 182 } 183 for _, fd := range fds { 184 if err := h.stor.ForceRemove(fd); err != nil { 185 h.t.Error("remove file: ", err) 186 } 187 } 188 } 189 190 func (h *dbCorruptHarness) removeOne(ft storage.FileType) { 191 fds, err := h.stor.List(ft) 192 if err != nil { 193 h.t.Fatal("get files: ", err) 194 } 195 fd := fds[rand.Intn(len(fds))] 196 h.t.Logf("removing file @%d", fd.Num) 197 if err := h.stor.Remove(fd); err != nil { 198 h.t.Error("remove file: ", err) 199 } 200 } 201 202 func (h *dbCorruptHarness) check(min, max int) { 203 p := &h.dbHarness 204 t := p.t 205 db := p.db 206 207 var n, badk, badv, missed, good int 208 iter := db.NewIterator(nil, p.ro) 209 for iter.Next() { 210 k := 0 211 fmt.Sscanf(string(iter.Key()), "%d", &k) 212 if k < n { 213 badk++ 214 continue 215 } 216 missed += k - n 217 n = k + 1 218 if !bytes.Equal(iter.Value(), tval(k, ctValSize)) { 219 badv++ 220 } else { 221 good++ 222 } 223 } 224 err := iter.Error() 225 iter.Release() 226 t.Logf("want=%d..%d got=%d badkeys=%d badvalues=%d missed=%d, err=%v", 227 min, max, good, badk, badv, missed, err) 228 if good < min || good > max { 229 t.Errorf("good entries number not in range") 230 } 231 } 232 233 func TestCorruptDB_Journal(t *testing.T) { 234 h := newDbCorruptHarness(t) 235 defer h.close() 236 237 h.build(100) 238 h.check(100, 100) 239 h.closeDB() 240 h.corrupt(storage.TypeJournal, -1, 19, 1) 241 h.corrupt(storage.TypeJournal, -1, 32*1024+1000, 1) 242 243 h.openDB() 244 h.check(36, 36) 245 } 246 247 func TestCorruptDB_Table(t *testing.T) { 248 h := newDbCorruptHarness(t) 249 defer h.close() 250 251 h.build(100) 252 h.compactMem() 253 h.compactRangeAt(0, "", "") 254 h.compactRangeAt(1, "", "") 255 h.closeDB() 256 h.corrupt(storage.TypeTable, -1, 100, 1) 257 258 h.openDB() 259 h.check(99, 99) 260 } 261 262 func TestCorruptDB_TableIndex(t *testing.T) { 263 h := newDbCorruptHarness(t) 264 defer h.close() 265 266 h.build(10000) 267 h.compactMem() 268 h.closeDB() 269 h.corrupt(storage.TypeTable, -1, -2000, 500) 270 271 h.openDB() 272 h.check(5000, 9999) 273 } 274 275 func TestCorruptDB_MissingManifest(t *testing.T) { 276 rnd := rand.New(rand.NewSource(0x0badda7a)) 277 h := newDbCorruptHarnessWopt(t, &opt.Options{ 278 BlockCacheCapacity: 100, 279 Strict: opt.StrictJournalChecksum, 280 WriteBuffer: 1000 * 60, 281 }) 282 defer h.close() 283 284 h.build(1000) 285 h.compactMem() 286 h.buildShuffled(1000, rnd) 287 h.compactMem() 288 h.deleteRand(500, 1000, rnd) 289 h.compactMem() 290 h.buildShuffled(1000, rnd) 291 h.compactMem() 292 h.deleteRand(500, 1000, rnd) 293 h.compactMem() 294 h.buildShuffled(1000, rnd) 295 h.compactMem() 296 h.closeDB() 297 298 h.forceRemoveAll(storage.TypeManifest) 299 h.openAssert(false) 300 301 h.recover() 302 h.check(1000, 1000) 303 h.build(1000) 304 h.compactMem() 305 h.compactRange("", "") 306 h.closeDB() 307 308 h.recover() 309 h.check(1000, 1000) 310 } 311 312 func TestCorruptDB_SequenceNumberRecovery(t *testing.T) { 313 h := newDbCorruptHarness(t) 314 defer h.close() 315 316 h.put("foo", "v1") 317 h.put("foo", "v2") 318 h.put("foo", "v3") 319 h.put("foo", "v4") 320 h.put("foo", "v5") 321 h.closeDB() 322 323 h.recover() 324 h.getVal("foo", "v5") 325 h.put("foo", "v6") 326 h.getVal("foo", "v6") 327 328 h.reopenDB() 329 h.getVal("foo", "v6") 330 } 331 332 func TestCorruptDB_SequenceNumberRecoveryTable(t *testing.T) { 333 h := newDbCorruptHarness(t) 334 defer h.close() 335 336 h.put("foo", "v1") 337 h.put("foo", "v2") 338 h.put("foo", "v3") 339 h.compactMem() 340 h.put("foo", "v4") 341 h.put("foo", "v5") 342 h.compactMem() 343 h.closeDB() 344 345 h.recover() 346 h.getVal("foo", "v5") 347 h.put("foo", "v6") 348 h.getVal("foo", "v6") 349 350 h.reopenDB() 351 h.getVal("foo", "v6") 352 } 353 354 func TestCorruptDB_CorruptedManifest(t *testing.T) { 355 h := newDbCorruptHarness(t) 356 defer h.close() 357 358 h.put("foo", "hello") 359 h.compactMem() 360 h.compactRange("", "") 361 h.closeDB() 362 h.corrupt(storage.TypeManifest, -1, 0, 1000) 363 h.openAssert(false) 364 365 h.recover() 366 h.getVal("foo", "hello") 367 } 368 369 func TestCorruptDB_CompactionInputError(t *testing.T) { 370 h := newDbCorruptHarness(t) 371 defer h.close() 372 373 h.build(10) 374 h.compactMem() 375 h.closeDB() 376 h.corrupt(storage.TypeTable, -1, 100, 1) 377 378 h.openDB() 379 h.check(9, 9) 380 381 h.build(10000) 382 h.check(10000, 10000) 383 } 384 385 func TestCorruptDB_UnrelatedKeys(t *testing.T) { 386 h := newDbCorruptHarness(t) 387 defer h.close() 388 389 h.build(10) 390 h.compactMem() 391 h.closeDB() 392 h.corrupt(storage.TypeTable, -1, 100, 1) 393 394 h.openDB() 395 h.put(string(tkey(1000)), string(tval(1000, ctValSize))) 396 h.getVal(string(tkey(1000)), string(tval(1000, ctValSize))) 397 h.compactMem() 398 h.getVal(string(tkey(1000)), string(tval(1000, ctValSize))) 399 } 400 401 func TestCorruptDB_Level0NewerFileHasOlderSeqnum(t *testing.T) { 402 h := newDbCorruptHarness(t) 403 defer h.close() 404 405 h.put("a", "v1") 406 h.put("b", "v1") 407 h.compactMem() 408 h.put("a", "v2") 409 h.put("b", "v2") 410 h.compactMem() 411 h.put("a", "v3") 412 h.put("b", "v3") 413 h.compactMem() 414 h.put("c", "v0") 415 h.put("d", "v0") 416 h.compactMem() 417 h.compactRangeAt(1, "", "") 418 h.closeDB() 419 420 h.recover() 421 h.getVal("a", "v3") 422 h.getVal("b", "v3") 423 h.getVal("c", "v0") 424 h.getVal("d", "v0") 425 } 426 427 func TestCorruptDB_RecoverInvalidSeq_Issue53(t *testing.T) { 428 h := newDbCorruptHarness(t) 429 defer h.close() 430 431 h.put("a", "v1") 432 h.put("b", "v1") 433 h.compactMem() 434 h.put("a", "v2") 435 h.put("b", "v2") 436 h.compactMem() 437 h.put("a", "v3") 438 h.put("b", "v3") 439 h.compactMem() 440 h.put("c", "v0") 441 h.put("d", "v0") 442 h.compactMem() 443 h.compactRangeAt(0, "", "") 444 h.closeDB() 445 446 h.recover() 447 h.getVal("a", "v3") 448 h.getVal("b", "v3") 449 h.getVal("c", "v0") 450 h.getVal("d", "v0") 451 } 452 453 func TestCorruptDB_MissingTableFiles(t *testing.T) { 454 h := newDbCorruptHarness(t) 455 defer h.close() 456 457 h.put("a", "v1") 458 h.put("b", "v1") 459 h.compactMem() 460 h.put("c", "v2") 461 h.put("d", "v2") 462 h.compactMem() 463 h.put("e", "v3") 464 h.put("f", "v3") 465 h.closeDB() 466 467 h.removeOne(storage.TypeTable) 468 h.openAssert(false) 469 } 470 471 func TestCorruptDB_RecoverTable(t *testing.T) { 472 h := newDbCorruptHarnessWopt(t, &opt.Options{ 473 WriteBuffer: 112 * opt.KiB, 474 CompactionTableSize: 90 * opt.KiB, 475 Filter: filter.NewBloomFilter(10), 476 }) 477 defer h.close() 478 479 h.build(1000) 480 h.compactMem() 481 h.compactRangeAt(0, "", "") 482 h.compactRangeAt(1, "", "") 483 seq := h.db.seq 484 h.closeDB() 485 h.corrupt(storage.TypeTable, 0, 1000, 1) 486 h.corrupt(storage.TypeTable, 3, 10000, 1) 487 // Corrupted filter shouldn't affect recovery. 488 h.corrupt(storage.TypeTable, 3, 113888, 10) 489 h.corrupt(storage.TypeTable, -1, 20000, 1) 490 491 h.recover() 492 if h.db.seq != seq { 493 t.Errorf("invalid seq, want=%d got=%d", seq, h.db.seq) 494 } 495 h.check(985, 985) 496 }