github.com/mckael/restic@v0.8.3/internal/repository/repository_test.go (about) 1 package repository_test 2 3 import ( 4 "bytes" 5 "context" 6 "crypto/sha256" 7 "io" 8 "math/rand" 9 "path/filepath" 10 "testing" 11 "time" 12 13 "github.com/restic/restic/internal/archiver" 14 "github.com/restic/restic/internal/repository" 15 "github.com/restic/restic/internal/restic" 16 rtest "github.com/restic/restic/internal/test" 17 ) 18 19 var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20} 20 21 var rnd = rand.New(rand.NewSource(time.Now().UnixNano())) 22 23 func TestSave(t *testing.T) { 24 repo, cleanup := repository.TestRepository(t) 25 defer cleanup() 26 27 for _, size := range testSizes { 28 data := make([]byte, size) 29 _, err := io.ReadFull(rnd, data) 30 rtest.OK(t, err) 31 32 id := restic.Hash(data) 33 34 // save 35 sid, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, restic.ID{}) 36 rtest.OK(t, err) 37 38 rtest.Equals(t, id, sid) 39 40 rtest.OK(t, repo.Flush(context.Background())) 41 // rtest.OK(t, repo.SaveIndex()) 42 43 // read back 44 buf := restic.NewBlobBuffer(size) 45 n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf) 46 rtest.OK(t, err) 47 rtest.Equals(t, len(buf), n) 48 49 rtest.Assert(t, len(buf) == len(data), 50 "number of bytes read back does not match: expected %d, got %d", 51 len(data), len(buf)) 52 53 rtest.Assert(t, bytes.Equal(buf, data), 54 "data does not match: expected %02x, got %02x", 55 data, buf) 56 } 57 } 58 59 func TestSaveFrom(t *testing.T) { 60 repo, cleanup := repository.TestRepository(t) 61 defer cleanup() 62 63 for _, size := range testSizes { 64 data := make([]byte, size) 65 _, err := io.ReadFull(rnd, data) 66 rtest.OK(t, err) 67 68 id := restic.Hash(data) 69 70 // save 71 id2, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, id) 72 rtest.OK(t, err) 73 rtest.Equals(t, id, id2) 74 75 rtest.OK(t, repo.Flush(context.Background())) 76 77 // read back 78 buf := restic.NewBlobBuffer(size) 79 n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf) 80 rtest.OK(t, err) 81 rtest.Equals(t, len(buf), n) 82 83 rtest.Assert(t, len(buf) == len(data), 84 "number of bytes read back does not match: expected %d, got %d", 85 len(data), len(buf)) 86 87 rtest.Assert(t, bytes.Equal(buf, data), 88 "data does not match: expected %02x, got %02x", 89 data, buf) 90 } 91 } 92 93 func BenchmarkSaveAndEncrypt(t *testing.B) { 94 repo, cleanup := repository.TestRepository(t) 95 defer cleanup() 96 97 size := 4 << 20 // 4MiB 98 99 data := make([]byte, size) 100 _, err := io.ReadFull(rnd, data) 101 rtest.OK(t, err) 102 103 id := restic.ID(sha256.Sum256(data)) 104 105 t.ResetTimer() 106 t.SetBytes(int64(size)) 107 108 for i := 0; i < t.N; i++ { 109 // save 110 _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, data, id) 111 rtest.OK(t, err) 112 } 113 } 114 115 func TestLoadTree(t *testing.T) { 116 repo, cleanup := repository.TestRepository(t) 117 defer cleanup() 118 119 if rtest.BenchArchiveDirectory == "" { 120 t.Skip("benchdir not set, skipping") 121 } 122 123 // archive a few files 124 sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil) 125 rtest.OK(t, repo.Flush(context.Background())) 126 127 _, err := repo.LoadTree(context.TODO(), *sn.Tree) 128 rtest.OK(t, err) 129 } 130 131 func BenchmarkLoadTree(t *testing.B) { 132 repo, cleanup := repository.TestRepository(t) 133 defer cleanup() 134 135 if rtest.BenchArchiveDirectory == "" { 136 t.Skip("benchdir not set, skipping") 137 } 138 139 // archive a few files 140 sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil) 141 rtest.OK(t, repo.Flush(context.Background())) 142 143 t.ResetTimer() 144 145 for i := 0; i < t.N; i++ { 146 _, err := repo.LoadTree(context.TODO(), *sn.Tree) 147 rtest.OK(t, err) 148 } 149 } 150 151 func TestLoadBlob(t *testing.T) { 152 repo, cleanup := repository.TestRepository(t) 153 defer cleanup() 154 155 length := 1000000 156 buf := restic.NewBlobBuffer(length) 157 _, err := io.ReadFull(rnd, buf) 158 rtest.OK(t, err) 159 160 id, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}) 161 rtest.OK(t, err) 162 rtest.OK(t, repo.Flush(context.Background())) 163 164 // first, test with buffers that are too small 165 for _, testlength := range []int{length - 20, length, restic.CiphertextLength(length) - 1} { 166 buf = make([]byte, 0, testlength) 167 n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf) 168 if err == nil { 169 t.Errorf("LoadBlob() did not return an error for a buffer that is too small to hold the blob") 170 continue 171 } 172 173 if n != 0 { 174 t.Errorf("LoadBlob() returned an error and n > 0") 175 continue 176 } 177 } 178 179 // then use buffers that are large enough 180 base := restic.CiphertextLength(length) 181 for _, testlength := range []int{base, base + 7, base + 15, base + 1000} { 182 buf = make([]byte, 0, testlength) 183 n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf) 184 if err != nil { 185 t.Errorf("LoadBlob() returned an error for buffer size %v: %v", testlength, err) 186 continue 187 } 188 189 if n != length { 190 t.Errorf("LoadBlob() returned the wrong number of bytes: want %v, got %v", length, n) 191 continue 192 } 193 } 194 } 195 196 func BenchmarkLoadBlob(b *testing.B) { 197 repo, cleanup := repository.TestRepository(b) 198 defer cleanup() 199 200 length := 1000000 201 buf := restic.NewBlobBuffer(length) 202 _, err := io.ReadFull(rnd, buf) 203 rtest.OK(b, err) 204 205 id, err := repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}) 206 rtest.OK(b, err) 207 rtest.OK(b, repo.Flush(context.Background())) 208 209 b.ResetTimer() 210 b.SetBytes(int64(length)) 211 212 for i := 0; i < b.N; i++ { 213 n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf) 214 rtest.OK(b, err) 215 if n != length { 216 b.Errorf("wanted %d bytes, got %d", length, n) 217 } 218 219 id2 := restic.Hash(buf[:n]) 220 if !id.Equal(id2) { 221 b.Errorf("wrong data returned, wanted %v, got %v", id.Str(), id2.Str()) 222 } 223 } 224 } 225 226 func BenchmarkLoadAndDecrypt(b *testing.B) { 227 repo, cleanup := repository.TestRepository(b) 228 defer cleanup() 229 230 length := 1000000 231 buf := restic.NewBlobBuffer(length) 232 _, err := io.ReadFull(rnd, buf) 233 rtest.OK(b, err) 234 235 dataID := restic.Hash(buf) 236 237 storageID, err := repo.SaveUnpacked(context.TODO(), restic.DataFile, buf) 238 rtest.OK(b, err) 239 // rtest.OK(b, repo.Flush()) 240 241 b.ResetTimer() 242 b.SetBytes(int64(length)) 243 244 for i := 0; i < b.N; i++ { 245 data, err := repo.LoadAndDecrypt(context.TODO(), restic.DataFile, storageID) 246 rtest.OK(b, err) 247 if len(data) != length { 248 b.Errorf("wanted %d bytes, got %d", length, len(data)) 249 } 250 251 id2 := restic.Hash(data) 252 if !dataID.Equal(id2) { 253 b.Errorf("wrong data returned, wanted %v, got %v", storageID.Str(), id2.Str()) 254 } 255 } 256 } 257 258 func TestLoadJSONUnpacked(t *testing.T) { 259 repo, cleanup := repository.TestRepository(t) 260 defer cleanup() 261 262 if rtest.BenchArchiveDirectory == "" { 263 t.Skip("benchdir not set, skipping") 264 } 265 266 // archive a snapshot 267 sn := restic.Snapshot{} 268 sn.Hostname = "foobar" 269 sn.Username = "test!" 270 271 id, err := repo.SaveJSONUnpacked(context.TODO(), restic.SnapshotFile, &sn) 272 rtest.OK(t, err) 273 274 var sn2 restic.Snapshot 275 276 // restore 277 err = repo.LoadJSONUnpacked(context.TODO(), restic.SnapshotFile, id, &sn2) 278 rtest.OK(t, err) 279 280 rtest.Equals(t, sn.Hostname, sn2.Hostname) 281 rtest.Equals(t, sn.Username, sn2.Username) 282 } 283 284 var repoFixture = filepath.Join("testdata", "test-repo.tar.gz") 285 286 func TestRepositoryLoadIndex(t *testing.T) { 287 repodir, cleanup := rtest.Env(t, repoFixture) 288 defer cleanup() 289 290 repo := repository.TestOpenLocal(t, repodir) 291 rtest.OK(t, repo.LoadIndex(context.TODO())) 292 } 293 294 func BenchmarkLoadIndex(b *testing.B) { 295 repository.TestUseLowSecurityKDFParameters(b) 296 297 repo, cleanup := repository.TestRepository(b) 298 defer cleanup() 299 300 idx := repository.NewIndex() 301 302 for i := 0; i < 5000; i++ { 303 idx.Store(restic.PackedBlob{ 304 Blob: restic.Blob{ 305 Type: restic.DataBlob, 306 Length: 1234, 307 ID: restic.NewRandomID(), 308 Offset: 1235, 309 }, 310 PackID: restic.NewRandomID(), 311 }) 312 } 313 314 id, err := repository.SaveIndex(context.TODO(), repo, idx) 315 rtest.OK(b, err) 316 317 b.Logf("index saved as %v (%v entries)", id.Str(), idx.Count(restic.DataBlob)) 318 fi, err := repo.Backend().Stat(context.TODO(), restic.Handle{Type: restic.IndexFile, Name: id.String()}) 319 rtest.OK(b, err) 320 b.Logf("filesize is %v", fi.Size) 321 322 b.ResetTimer() 323 324 for i := 0; i < b.N; i++ { 325 _, err := repository.LoadIndex(context.TODO(), repo, id) 326 rtest.OK(b, err) 327 } 328 } 329 330 // saveRandomDataBlobs generates random data blobs and saves them to the repository. 331 func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax int) { 332 for i := 0; i < num; i++ { 333 size := rand.Int() % sizeMax 334 335 buf := make([]byte, size) 336 _, err := io.ReadFull(rnd, buf) 337 rtest.OK(t, err) 338 339 _, err = repo.SaveBlob(context.TODO(), restic.DataBlob, buf, restic.ID{}) 340 rtest.OK(t, err) 341 } 342 } 343 344 func TestRepositoryIncrementalIndex(t *testing.T) { 345 repo, cleanup := repository.TestRepository(t) 346 defer cleanup() 347 348 repository.IndexFull = func(*repository.Index) bool { return true } 349 350 // add 15 packs 351 for j := 0; j < 5; j++ { 352 // add 3 packs, write intermediate index 353 for i := 0; i < 3; i++ { 354 saveRandomDataBlobs(t, repo, 5, 1<<15) 355 rtest.OK(t, repo.Flush(context.Background())) 356 } 357 358 rtest.OK(t, repo.SaveFullIndex(context.TODO())) 359 } 360 361 // add another 5 packs 362 for i := 0; i < 5; i++ { 363 saveRandomDataBlobs(t, repo, 5, 1<<15) 364 rtest.OK(t, repo.Flush(context.Background())) 365 } 366 367 // save final index 368 rtest.OK(t, repo.SaveIndex(context.TODO())) 369 370 packEntries := make(map[restic.ID]map[restic.ID]struct{}) 371 372 err := repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error { 373 idx, err := repository.LoadIndex(context.TODO(), repo, id) 374 rtest.OK(t, err) 375 376 for pb := range idx.Each(context.TODO()) { 377 if _, ok := packEntries[pb.PackID]; !ok { 378 packEntries[pb.PackID] = make(map[restic.ID]struct{}) 379 } 380 381 packEntries[pb.PackID][id] = struct{}{} 382 } 383 return nil 384 }) 385 if err != nil { 386 t.Fatal(err) 387 } 388 389 for packID, ids := range packEntries { 390 if len(ids) > 1 { 391 t.Errorf("pack %v listed in %d indexes\n", packID, len(ids)) 392 } 393 } 394 }