github.com/advanderveer/restic@v0.8.1-0.20171209104529-42a8c19aaea6/internal/checker/checker_test.go (about) 1 package checker_test 2 3 import ( 4 "context" 5 "io" 6 "math/rand" 7 "path/filepath" 8 "sort" 9 "testing" 10 "time" 11 12 "github.com/restic/restic/internal/archiver" 13 "github.com/restic/restic/internal/checker" 14 "github.com/restic/restic/internal/repository" 15 "github.com/restic/restic/internal/restic" 16 "github.com/restic/restic/internal/test" 17 ) 18 19 var checkerTestData = filepath.Join("testdata", "checker-test-repo.tar.gz") 20 21 func collectErrors(ctx context.Context, f func(context.Context, chan<- error)) (errs []error) { 22 ctx, cancel := context.WithCancel(ctx) 23 defer cancel() 24 25 errChan := make(chan error) 26 27 go f(ctx, errChan) 28 29 for err := range errChan { 30 errs = append(errs, err) 31 } 32 33 return errs 34 } 35 36 func checkPacks(chkr *checker.Checker) []error { 37 return collectErrors(context.TODO(), chkr.Packs) 38 } 39 40 func checkStruct(chkr *checker.Checker) []error { 41 return collectErrors(context.TODO(), chkr.Structure) 42 } 43 44 func checkData(chkr *checker.Checker) []error { 45 return collectErrors( 46 context.TODO(), 47 func(ctx context.Context, errCh chan<- error) { 48 chkr.ReadData(ctx, nil, errCh) 49 }, 50 ) 51 } 52 53 func TestCheckRepo(t *testing.T) { 54 repodir, cleanup := test.Env(t, checkerTestData) 55 defer cleanup() 56 57 repo := repository.TestOpenLocal(t, repodir) 58 59 chkr := checker.New(repo) 60 hints, errs := chkr.LoadIndex(context.TODO()) 61 if len(errs) > 0 { 62 t.Fatalf("expected no errors, got %v: %v", len(errs), errs) 63 } 64 65 if len(hints) > 0 { 66 t.Errorf("expected no hints, got %v: %v", len(hints), hints) 67 } 68 69 test.OKs(t, checkPacks(chkr)) 70 test.OKs(t, checkStruct(chkr)) 71 } 72 73 func TestMissingPack(t *testing.T) { 74 repodir, cleanup := test.Env(t, checkerTestData) 75 defer cleanup() 76 77 repo := repository.TestOpenLocal(t, repodir) 78 79 packHandle := restic.Handle{ 80 Type: restic.DataFile, 81 Name: "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6", 82 } 83 test.OK(t, repo.Backend().Remove(context.TODO(), packHandle)) 84 85 chkr := checker.New(repo) 86 hints, errs := chkr.LoadIndex(context.TODO()) 87 if len(errs) > 0 { 88 t.Fatalf("expected no errors, got %v: %v", len(errs), errs) 89 } 90 91 if len(hints) > 0 { 92 t.Errorf("expected no hints, got %v: %v", len(hints), hints) 93 } 94 95 errs = checkPacks(chkr) 96 97 test.Assert(t, len(errs) == 1, 98 "expected exactly one error, got %v", len(errs)) 99 100 if err, ok := errs[0].(checker.PackError); ok { 101 test.Equals(t, packHandle.Name, err.ID.String()) 102 } else { 103 t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err) 104 } 105 } 106 107 func TestUnreferencedPack(t *testing.T) { 108 repodir, cleanup := test.Env(t, checkerTestData) 109 defer cleanup() 110 111 repo := repository.TestOpenLocal(t, repodir) 112 113 // index 3f1a only references pack 60e0 114 packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e" 115 indexHandle := restic.Handle{ 116 Type: restic.IndexFile, 117 Name: "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44", 118 } 119 test.OK(t, repo.Backend().Remove(context.TODO(), indexHandle)) 120 121 chkr := checker.New(repo) 122 hints, errs := chkr.LoadIndex(context.TODO()) 123 if len(errs) > 0 { 124 t.Fatalf("expected no errors, got %v: %v", len(errs), errs) 125 } 126 127 if len(hints) > 0 { 128 t.Errorf("expected no hints, got %v: %v", len(hints), hints) 129 } 130 131 errs = checkPacks(chkr) 132 133 test.Assert(t, len(errs) == 1, 134 "expected exactly one error, got %v", len(errs)) 135 136 if err, ok := errs[0].(checker.PackError); ok { 137 test.Equals(t, packID, err.ID.String()) 138 } else { 139 t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err) 140 } 141 } 142 143 func TestUnreferencedBlobs(t *testing.T) { 144 repodir, cleanup := test.Env(t, checkerTestData) 145 defer cleanup() 146 147 repo := repository.TestOpenLocal(t, repodir) 148 149 snapshotHandle := restic.Handle{ 150 Type: restic.SnapshotFile, 151 Name: "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02", 152 } 153 test.OK(t, repo.Backend().Remove(context.TODO(), snapshotHandle)) 154 155 unusedBlobsBySnapshot := restic.IDs{ 156 restic.TestParseID("58c748bbe2929fdf30c73262bd8313fe828f8925b05d1d4a87fe109082acb849"), 157 restic.TestParseID("988a272ab9768182abfd1fe7d7a7b68967825f0b861d3b36156795832c772235"), 158 restic.TestParseID("c01952de4d91da1b1b80bc6e06eaa4ec21523f4853b69dc8231708b9b7ec62d8"), 159 restic.TestParseID("bec3a53d7dc737f9a9bee68b107ec9e8ad722019f649b34d474b9982c3a3fec7"), 160 restic.TestParseID("2a6f01e5e92d8343c4c6b78b51c5a4dc9c39d42c04e26088c7614b13d8d0559d"), 161 restic.TestParseID("18b51b327df9391732ba7aaf841a4885f350d8a557b2da8352c9acf8898e3f10"), 162 } 163 164 sort.Sort(unusedBlobsBySnapshot) 165 166 chkr := checker.New(repo) 167 hints, errs := chkr.LoadIndex(context.TODO()) 168 if len(errs) > 0 { 169 t.Fatalf("expected no errors, got %v: %v", len(errs), errs) 170 } 171 172 if len(hints) > 0 { 173 t.Errorf("expected no hints, got %v: %v", len(hints), hints) 174 } 175 176 test.OKs(t, checkPacks(chkr)) 177 test.OKs(t, checkStruct(chkr)) 178 179 blobs := chkr.UnusedBlobs() 180 sort.Sort(blobs) 181 182 test.Equals(t, unusedBlobsBySnapshot, blobs) 183 } 184 185 func TestModifiedIndex(t *testing.T) { 186 repodir, cleanup := test.Env(t, checkerTestData) 187 defer cleanup() 188 189 repo := repository.TestOpenLocal(t, repodir) 190 191 done := make(chan struct{}) 192 defer close(done) 193 194 h := restic.Handle{ 195 Type: restic.IndexFile, 196 Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd", 197 } 198 f, err := repo.Backend().Load(context.TODO(), h, 0, 0) 199 test.OK(t, err) 200 201 // save the index again with a modified name so that the hash doesn't match 202 // the content any more 203 h2 := restic.Handle{ 204 Type: restic.IndexFile, 205 Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd", 206 } 207 err = repo.Backend().Save(context.TODO(), h2, f) 208 test.OK(t, err) 209 210 test.OK(t, f.Close()) 211 212 chkr := checker.New(repo) 213 hints, errs := chkr.LoadIndex(context.TODO()) 214 if len(errs) == 0 { 215 t.Fatalf("expected errors not found") 216 } 217 218 for _, err := range errs { 219 t.Logf("found expected error %v", err) 220 } 221 222 if len(hints) > 0 { 223 t.Errorf("expected no hints, got %v: %v", len(hints), hints) 224 } 225 } 226 227 var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz") 228 229 func TestDuplicatePacksInIndex(t *testing.T) { 230 repodir, cleanup := test.Env(t, checkerDuplicateIndexTestData) 231 defer cleanup() 232 233 repo := repository.TestOpenLocal(t, repodir) 234 235 chkr := checker.New(repo) 236 hints, errs := chkr.LoadIndex(context.TODO()) 237 if len(hints) == 0 { 238 t.Fatalf("did not get expected checker hints for duplicate packs in indexes") 239 } 240 241 found := false 242 for _, hint := range hints { 243 if _, ok := hint.(checker.ErrDuplicatePacks); ok { 244 found = true 245 } else { 246 t.Errorf("got unexpected hint: %v", hint) 247 } 248 } 249 250 if !found { 251 t.Fatalf("did not find hint ErrDuplicatePacks") 252 } 253 254 if len(errs) > 0 { 255 t.Errorf("expected no errors, got %v: %v", len(errs), errs) 256 } 257 } 258 259 // errorBackend randomly modifies data after reading. 260 type errorBackend struct { 261 restic.Backend 262 ProduceErrors bool 263 } 264 265 func (b errorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { 266 rd, err := b.Backend.Load(ctx, h, length, offset) 267 if err != nil { 268 return rd, err 269 } 270 271 if b.ProduceErrors { 272 return errorReadCloser{rd}, err 273 } 274 275 return rd, nil 276 } 277 278 type errorReadCloser struct { 279 io.ReadCloser 280 } 281 282 func (erd errorReadCloser) Read(p []byte) (int, error) { 283 n, err := erd.ReadCloser.Read(p) 284 if n > 0 { 285 induceError(p[:n]) 286 } 287 return n, err 288 } 289 290 func (erd errorReadCloser) Close() error { 291 return erd.ReadCloser.Close() 292 } 293 294 // induceError flips a bit in the slice. 295 func induceError(data []byte) { 296 if rand.Float32() < 0.2 { 297 return 298 } 299 300 pos := rand.Intn(len(data)) 301 data[pos] ^= 1 302 } 303 304 func TestCheckerModifiedData(t *testing.T) { 305 repo, cleanup := repository.TestRepository(t) 306 defer cleanup() 307 308 arch := archiver.New(repo) 309 _, id, err := arch.Snapshot(context.TODO(), nil, []string{"."}, nil, "localhost", nil, time.Now()) 310 test.OK(t, err) 311 t.Logf("archived as %v", id.Str()) 312 313 beError := &errorBackend{Backend: repo.Backend()} 314 checkRepo := repository.New(beError) 315 test.OK(t, checkRepo.SearchKey(context.TODO(), test.TestPassword, 5)) 316 317 chkr := checker.New(checkRepo) 318 319 hints, errs := chkr.LoadIndex(context.TODO()) 320 if len(errs) > 0 { 321 t.Fatalf("expected no errors, got %v: %v", len(errs), errs) 322 } 323 324 if len(hints) > 0 { 325 t.Errorf("expected no hints, got %v: %v", len(hints), hints) 326 } 327 328 beError.ProduceErrors = true 329 errFound := false 330 for _, err := range checkPacks(chkr) { 331 t.Logf("pack error: %v", err) 332 } 333 334 for _, err := range checkStruct(chkr) { 335 t.Logf("struct error: %v", err) 336 } 337 338 for _, err := range checkData(chkr) { 339 t.Logf("data error: %v", err) 340 errFound = true 341 } 342 343 if !errFound { 344 t.Fatal("no error found, checker is broken") 345 } 346 } 347 348 func BenchmarkChecker(t *testing.B) { 349 repodir, cleanup := test.Env(t, checkerTestData) 350 defer cleanup() 351 352 repo := repository.TestOpenLocal(t, repodir) 353 354 chkr := checker.New(repo) 355 hints, errs := chkr.LoadIndex(context.TODO()) 356 if len(errs) > 0 { 357 t.Fatalf("expected no errors, got %v: %v", len(errs), errs) 358 } 359 360 if len(hints) > 0 { 361 t.Errorf("expected no hints, got %v: %v", len(hints), hints) 362 } 363 364 t.ResetTimer() 365 366 for i := 0; i < t.N; i++ { 367 test.OKs(t, checkPacks(chkr)) 368 test.OKs(t, checkStruct(chkr)) 369 test.OKs(t, checkData(chkr)) 370 } 371 }