github.com/advanderveer/restic@v0.8.1-0.20171209104529-42a8c19aaea6/internal/checker/checker_test.go (about)

     1  package checker_test
     2  
     3  import (
     4  	"context"
     5  	"io"
     6  	"math/rand"
     7  	"path/filepath"
     8  	"sort"
     9  	"testing"
    10  	"time"
    11  
    12  	"github.com/restic/restic/internal/archiver"
    13  	"github.com/restic/restic/internal/checker"
    14  	"github.com/restic/restic/internal/repository"
    15  	"github.com/restic/restic/internal/restic"
    16  	"github.com/restic/restic/internal/test"
    17  )
    18  
    19  var checkerTestData = filepath.Join("testdata", "checker-test-repo.tar.gz")
    20  
    21  func collectErrors(ctx context.Context, f func(context.Context, chan<- error)) (errs []error) {
    22  	ctx, cancel := context.WithCancel(ctx)
    23  	defer cancel()
    24  
    25  	errChan := make(chan error)
    26  
    27  	go f(ctx, errChan)
    28  
    29  	for err := range errChan {
    30  		errs = append(errs, err)
    31  	}
    32  
    33  	return errs
    34  }
    35  
    36  func checkPacks(chkr *checker.Checker) []error {
    37  	return collectErrors(context.TODO(), chkr.Packs)
    38  }
    39  
    40  func checkStruct(chkr *checker.Checker) []error {
    41  	return collectErrors(context.TODO(), chkr.Structure)
    42  }
    43  
    44  func checkData(chkr *checker.Checker) []error {
    45  	return collectErrors(
    46  		context.TODO(),
    47  		func(ctx context.Context, errCh chan<- error) {
    48  			chkr.ReadData(ctx, nil, errCh)
    49  		},
    50  	)
    51  }
    52  
    53  func TestCheckRepo(t *testing.T) {
    54  	repodir, cleanup := test.Env(t, checkerTestData)
    55  	defer cleanup()
    56  
    57  	repo := repository.TestOpenLocal(t, repodir)
    58  
    59  	chkr := checker.New(repo)
    60  	hints, errs := chkr.LoadIndex(context.TODO())
    61  	if len(errs) > 0 {
    62  		t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
    63  	}
    64  
    65  	if len(hints) > 0 {
    66  		t.Errorf("expected no hints, got %v: %v", len(hints), hints)
    67  	}
    68  
    69  	test.OKs(t, checkPacks(chkr))
    70  	test.OKs(t, checkStruct(chkr))
    71  }
    72  
    73  func TestMissingPack(t *testing.T) {
    74  	repodir, cleanup := test.Env(t, checkerTestData)
    75  	defer cleanup()
    76  
    77  	repo := repository.TestOpenLocal(t, repodir)
    78  
    79  	packHandle := restic.Handle{
    80  		Type: restic.DataFile,
    81  		Name: "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6",
    82  	}
    83  	test.OK(t, repo.Backend().Remove(context.TODO(), packHandle))
    84  
    85  	chkr := checker.New(repo)
    86  	hints, errs := chkr.LoadIndex(context.TODO())
    87  	if len(errs) > 0 {
    88  		t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
    89  	}
    90  
    91  	if len(hints) > 0 {
    92  		t.Errorf("expected no hints, got %v: %v", len(hints), hints)
    93  	}
    94  
    95  	errs = checkPacks(chkr)
    96  
    97  	test.Assert(t, len(errs) == 1,
    98  		"expected exactly one error, got %v", len(errs))
    99  
   100  	if err, ok := errs[0].(checker.PackError); ok {
   101  		test.Equals(t, packHandle.Name, err.ID.String())
   102  	} else {
   103  		t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
   104  	}
   105  }
   106  
   107  func TestUnreferencedPack(t *testing.T) {
   108  	repodir, cleanup := test.Env(t, checkerTestData)
   109  	defer cleanup()
   110  
   111  	repo := repository.TestOpenLocal(t, repodir)
   112  
   113  	// index 3f1a only references pack 60e0
   114  	packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"
   115  	indexHandle := restic.Handle{
   116  		Type: restic.IndexFile,
   117  		Name: "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44",
   118  	}
   119  	test.OK(t, repo.Backend().Remove(context.TODO(), indexHandle))
   120  
   121  	chkr := checker.New(repo)
   122  	hints, errs := chkr.LoadIndex(context.TODO())
   123  	if len(errs) > 0 {
   124  		t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
   125  	}
   126  
   127  	if len(hints) > 0 {
   128  		t.Errorf("expected no hints, got %v: %v", len(hints), hints)
   129  	}
   130  
   131  	errs = checkPacks(chkr)
   132  
   133  	test.Assert(t, len(errs) == 1,
   134  		"expected exactly one error, got %v", len(errs))
   135  
   136  	if err, ok := errs[0].(checker.PackError); ok {
   137  		test.Equals(t, packID, err.ID.String())
   138  	} else {
   139  		t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err)
   140  	}
   141  }
   142  
   143  func TestUnreferencedBlobs(t *testing.T) {
   144  	repodir, cleanup := test.Env(t, checkerTestData)
   145  	defer cleanup()
   146  
   147  	repo := repository.TestOpenLocal(t, repodir)
   148  
   149  	snapshotHandle := restic.Handle{
   150  		Type: restic.SnapshotFile,
   151  		Name: "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02",
   152  	}
   153  	test.OK(t, repo.Backend().Remove(context.TODO(), snapshotHandle))
   154  
   155  	unusedBlobsBySnapshot := restic.IDs{
   156  		restic.TestParseID("58c748bbe2929fdf30c73262bd8313fe828f8925b05d1d4a87fe109082acb849"),
   157  		restic.TestParseID("988a272ab9768182abfd1fe7d7a7b68967825f0b861d3b36156795832c772235"),
   158  		restic.TestParseID("c01952de4d91da1b1b80bc6e06eaa4ec21523f4853b69dc8231708b9b7ec62d8"),
   159  		restic.TestParseID("bec3a53d7dc737f9a9bee68b107ec9e8ad722019f649b34d474b9982c3a3fec7"),
   160  		restic.TestParseID("2a6f01e5e92d8343c4c6b78b51c5a4dc9c39d42c04e26088c7614b13d8d0559d"),
   161  		restic.TestParseID("18b51b327df9391732ba7aaf841a4885f350d8a557b2da8352c9acf8898e3f10"),
   162  	}
   163  
   164  	sort.Sort(unusedBlobsBySnapshot)
   165  
   166  	chkr := checker.New(repo)
   167  	hints, errs := chkr.LoadIndex(context.TODO())
   168  	if len(errs) > 0 {
   169  		t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
   170  	}
   171  
   172  	if len(hints) > 0 {
   173  		t.Errorf("expected no hints, got %v: %v", len(hints), hints)
   174  	}
   175  
   176  	test.OKs(t, checkPacks(chkr))
   177  	test.OKs(t, checkStruct(chkr))
   178  
   179  	blobs := chkr.UnusedBlobs()
   180  	sort.Sort(blobs)
   181  
   182  	test.Equals(t, unusedBlobsBySnapshot, blobs)
   183  }
   184  
   185  func TestModifiedIndex(t *testing.T) {
   186  	repodir, cleanup := test.Env(t, checkerTestData)
   187  	defer cleanup()
   188  
   189  	repo := repository.TestOpenLocal(t, repodir)
   190  
   191  	done := make(chan struct{})
   192  	defer close(done)
   193  
   194  	h := restic.Handle{
   195  		Type: restic.IndexFile,
   196  		Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
   197  	}
   198  	f, err := repo.Backend().Load(context.TODO(), h, 0, 0)
   199  	test.OK(t, err)
   200  
   201  	// save the index again with a modified name so that the hash doesn't match
   202  	// the content any more
   203  	h2 := restic.Handle{
   204  		Type: restic.IndexFile,
   205  		Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
   206  	}
   207  	err = repo.Backend().Save(context.TODO(), h2, f)
   208  	test.OK(t, err)
   209  
   210  	test.OK(t, f.Close())
   211  
   212  	chkr := checker.New(repo)
   213  	hints, errs := chkr.LoadIndex(context.TODO())
   214  	if len(errs) == 0 {
   215  		t.Fatalf("expected errors not found")
   216  	}
   217  
   218  	for _, err := range errs {
   219  		t.Logf("found expected error %v", err)
   220  	}
   221  
   222  	if len(hints) > 0 {
   223  		t.Errorf("expected no hints, got %v: %v", len(hints), hints)
   224  	}
   225  }
   226  
   227  var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz")
   228  
   229  func TestDuplicatePacksInIndex(t *testing.T) {
   230  	repodir, cleanup := test.Env(t, checkerDuplicateIndexTestData)
   231  	defer cleanup()
   232  
   233  	repo := repository.TestOpenLocal(t, repodir)
   234  
   235  	chkr := checker.New(repo)
   236  	hints, errs := chkr.LoadIndex(context.TODO())
   237  	if len(hints) == 0 {
   238  		t.Fatalf("did not get expected checker hints for duplicate packs in indexes")
   239  	}
   240  
   241  	found := false
   242  	for _, hint := range hints {
   243  		if _, ok := hint.(checker.ErrDuplicatePacks); ok {
   244  			found = true
   245  		} else {
   246  			t.Errorf("got unexpected hint: %v", hint)
   247  		}
   248  	}
   249  
   250  	if !found {
   251  		t.Fatalf("did not find hint ErrDuplicatePacks")
   252  	}
   253  
   254  	if len(errs) > 0 {
   255  		t.Errorf("expected no errors, got %v: %v", len(errs), errs)
   256  	}
   257  }
   258  
   259  // errorBackend randomly modifies data after reading.
   260  type errorBackend struct {
   261  	restic.Backend
   262  	ProduceErrors bool
   263  }
   264  
   265  func (b errorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
   266  	rd, err := b.Backend.Load(ctx, h, length, offset)
   267  	if err != nil {
   268  		return rd, err
   269  	}
   270  
   271  	if b.ProduceErrors {
   272  		return errorReadCloser{rd}, err
   273  	}
   274  
   275  	return rd, nil
   276  }
   277  
   278  type errorReadCloser struct {
   279  	io.ReadCloser
   280  }
   281  
   282  func (erd errorReadCloser) Read(p []byte) (int, error) {
   283  	n, err := erd.ReadCloser.Read(p)
   284  	if n > 0 {
   285  		induceError(p[:n])
   286  	}
   287  	return n, err
   288  }
   289  
   290  func (erd errorReadCloser) Close() error {
   291  	return erd.ReadCloser.Close()
   292  }
   293  
   294  // induceError flips a bit in the slice.
   295  func induceError(data []byte) {
   296  	if rand.Float32() < 0.2 {
   297  		return
   298  	}
   299  
   300  	pos := rand.Intn(len(data))
   301  	data[pos] ^= 1
   302  }
   303  
   304  func TestCheckerModifiedData(t *testing.T) {
   305  	repo, cleanup := repository.TestRepository(t)
   306  	defer cleanup()
   307  
   308  	arch := archiver.New(repo)
   309  	_, id, err := arch.Snapshot(context.TODO(), nil, []string{"."}, nil, "localhost", nil, time.Now())
   310  	test.OK(t, err)
   311  	t.Logf("archived as %v", id.Str())
   312  
   313  	beError := &errorBackend{Backend: repo.Backend()}
   314  	checkRepo := repository.New(beError)
   315  	test.OK(t, checkRepo.SearchKey(context.TODO(), test.TestPassword, 5))
   316  
   317  	chkr := checker.New(checkRepo)
   318  
   319  	hints, errs := chkr.LoadIndex(context.TODO())
   320  	if len(errs) > 0 {
   321  		t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
   322  	}
   323  
   324  	if len(hints) > 0 {
   325  		t.Errorf("expected no hints, got %v: %v", len(hints), hints)
   326  	}
   327  
   328  	beError.ProduceErrors = true
   329  	errFound := false
   330  	for _, err := range checkPacks(chkr) {
   331  		t.Logf("pack error: %v", err)
   332  	}
   333  
   334  	for _, err := range checkStruct(chkr) {
   335  		t.Logf("struct error: %v", err)
   336  	}
   337  
   338  	for _, err := range checkData(chkr) {
   339  		t.Logf("data error: %v", err)
   340  		errFound = true
   341  	}
   342  
   343  	if !errFound {
   344  		t.Fatal("no error found, checker is broken")
   345  	}
   346  }
   347  
   348  func BenchmarkChecker(t *testing.B) {
   349  	repodir, cleanup := test.Env(t, checkerTestData)
   350  	defer cleanup()
   351  
   352  	repo := repository.TestOpenLocal(t, repodir)
   353  
   354  	chkr := checker.New(repo)
   355  	hints, errs := chkr.LoadIndex(context.TODO())
   356  	if len(errs) > 0 {
   357  		t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
   358  	}
   359  
   360  	if len(hints) > 0 {
   361  		t.Errorf("expected no hints, got %v: %v", len(hints), hints)
   362  	}
   363  
   364  	t.ResetTimer()
   365  
   366  	for i := 0; i < t.N; i++ {
   367  		test.OKs(t, checkPacks(chkr))
   368  		test.OKs(t, checkStruct(chkr))
   369  		test.OKs(t, checkData(chkr))
   370  	}
   371  }