github.com/mckael/restic@v0.8.3/internal/repository/index_test.go (about) 1 package repository_test 2 3 import ( 4 "bytes" 5 "math/rand" 6 "testing" 7 8 "github.com/restic/restic/internal/repository" 9 "github.com/restic/restic/internal/restic" 10 rtest "github.com/restic/restic/internal/test" 11 ) 12 13 func TestIndexSerialize(t *testing.T) { 14 type testEntry struct { 15 id restic.ID 16 pack restic.ID 17 tpe restic.BlobType 18 offset, length uint 19 } 20 tests := []testEntry{} 21 22 idx := repository.NewIndex() 23 24 // create 50 packs with 20 blobs each 25 for i := 0; i < 50; i++ { 26 packID := restic.NewRandomID() 27 28 pos := uint(0) 29 for j := 0; j < 20; j++ { 30 id := restic.NewRandomID() 31 length := uint(i*100 + j) 32 idx.Store(restic.PackedBlob{ 33 Blob: restic.Blob{ 34 Type: restic.DataBlob, 35 ID: id, 36 Offset: pos, 37 Length: length, 38 }, 39 PackID: packID, 40 }) 41 42 tests = append(tests, testEntry{ 43 id: id, 44 pack: packID, 45 tpe: restic.DataBlob, 46 offset: pos, 47 length: length, 48 }) 49 50 pos += length 51 } 52 } 53 54 wr := bytes.NewBuffer(nil) 55 err := idx.Encode(wr) 56 rtest.OK(t, err) 57 58 idx2, err := repository.DecodeIndex(wr.Bytes()) 59 rtest.OK(t, err) 60 rtest.Assert(t, idx2 != nil, 61 "nil returned for decoded index") 62 63 wr2 := bytes.NewBuffer(nil) 64 err = idx2.Encode(wr2) 65 rtest.OK(t, err) 66 67 for _, testBlob := range tests { 68 list, found := idx.Lookup(testBlob.id, testBlob.tpe) 69 rtest.Assert(t, found, "Expected to find blob id %v", testBlob.id.Str()) 70 71 if len(list) != 1 { 72 t.Errorf("expected one result for blob %v, got %v: %v", testBlob.id.Str(), len(list), list) 73 } 74 result := list[0] 75 76 rtest.Equals(t, testBlob.pack, result.PackID) 77 rtest.Equals(t, testBlob.tpe, result.Type) 78 rtest.Equals(t, testBlob.offset, result.Offset) 79 rtest.Equals(t, testBlob.length, result.Length) 80 81 list2, found := idx2.Lookup(testBlob.id, testBlob.tpe) 82 rtest.Assert(t, found, "Expected to find blob id %v", testBlob.id) 83 84 if len(list2) != 1 { 85 t.Errorf("expected one result for blob %v, got %v: %v", testBlob.id.Str(), len(list2), list2) 86 } 87 result2 := list2[0] 88 89 rtest.Equals(t, testBlob.pack, result2.PackID) 90 rtest.Equals(t, testBlob.tpe, result2.Type) 91 rtest.Equals(t, testBlob.offset, result2.Offset) 92 rtest.Equals(t, testBlob.length, result2.Length) 93 } 94 95 // add more blobs to idx 96 newtests := []testEntry{} 97 for i := 0; i < 10; i++ { 98 packID := restic.NewRandomID() 99 100 pos := uint(0) 101 for j := 0; j < 10; j++ { 102 id := restic.NewRandomID() 103 length := uint(i*100 + j) 104 idx.Store(restic.PackedBlob{ 105 Blob: restic.Blob{ 106 Type: restic.DataBlob, 107 ID: id, 108 Offset: pos, 109 Length: length, 110 }, 111 PackID: packID, 112 }) 113 114 newtests = append(newtests, testEntry{ 115 id: id, 116 pack: packID, 117 tpe: restic.DataBlob, 118 offset: pos, 119 length: length, 120 }) 121 122 pos += length 123 } 124 } 125 126 // serialize idx, unserialize to idx3 127 wr3 := bytes.NewBuffer(nil) 128 err = idx.Finalize(wr3) 129 rtest.OK(t, err) 130 131 rtest.Assert(t, idx.Final(), 132 "index not final after encoding") 133 134 id := restic.NewRandomID() 135 rtest.OK(t, idx.SetID(id)) 136 id2, err := idx.ID() 137 rtest.Assert(t, id2.Equal(id), 138 "wrong ID returned: want %v, got %v", id, id2) 139 140 idx3, err := repository.DecodeIndex(wr3.Bytes()) 141 rtest.OK(t, err) 142 rtest.Assert(t, idx3 != nil, 143 "nil returned for decoded index") 144 rtest.Assert(t, idx3.Final(), 145 "decoded index is not final") 146 147 // all new blobs must be in the index 148 for _, testBlob := range newtests { 149 list, found := idx3.Lookup(testBlob.id, testBlob.tpe) 150 rtest.Assert(t, found, "Expected to find blob id %v", testBlob.id.Str()) 151 152 if len(list) != 1 { 153 t.Errorf("expected one result for blob %v, got %v: %v", testBlob.id.Str(), len(list), list) 154 } 155 156 blob := list[0] 157 158 rtest.Equals(t, testBlob.pack, blob.PackID) 159 rtest.Equals(t, testBlob.tpe, blob.Type) 160 rtest.Equals(t, testBlob.offset, blob.Offset) 161 rtest.Equals(t, testBlob.length, blob.Length) 162 } 163 } 164 165 func TestIndexSize(t *testing.T) { 166 idx := repository.NewIndex() 167 168 packs := 200 169 blobs := 100 170 for i := 0; i < packs; i++ { 171 packID := restic.NewRandomID() 172 173 pos := uint(0) 174 for j := 0; j < blobs; j++ { 175 id := restic.NewRandomID() 176 length := uint(i*100 + j) 177 idx.Store(restic.PackedBlob{ 178 Blob: restic.Blob{ 179 Type: restic.DataBlob, 180 ID: id, 181 Offset: pos, 182 Length: length, 183 }, 184 PackID: packID, 185 }) 186 187 pos += length 188 } 189 } 190 191 wr := bytes.NewBuffer(nil) 192 193 err := idx.Encode(wr) 194 rtest.OK(t, err) 195 196 t.Logf("Index file size for %d blobs in %d packs is %d", blobs*packs, packs, wr.Len()) 197 } 198 199 // example index serialization from doc/Design.rst 200 var docExample = []byte(` 201 { 202 "supersedes": [ 203 "ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452" 204 ], 205 "packs": [ 206 { 207 "id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c", 208 "blobs": [ 209 { 210 "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce", 211 "type": "data", 212 "offset": 0, 213 "length": 25 214 },{ 215 "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae", 216 "type": "tree", 217 "offset": 38, 218 "length": 100 219 }, 220 { 221 "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66", 222 "type": "data", 223 "offset": 150, 224 "length": 123 225 } 226 ] 227 } 228 ] 229 } 230 `) 231 232 var docOldExample = []byte(` 233 [ { 234 "id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c", 235 "blobs": [ 236 { 237 "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce", 238 "type": "data", 239 "offset": 0, 240 "length": 25 241 },{ 242 "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae", 243 "type": "tree", 244 "offset": 38, 245 "length": 100 246 }, 247 { 248 "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66", 249 "type": "data", 250 "offset": 150, 251 "length": 123 252 } 253 ] 254 } ] 255 `) 256 257 var exampleTests = []struct { 258 id, packID restic.ID 259 tpe restic.BlobType 260 offset, length uint 261 }{ 262 { 263 restic.TestParseID("3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce"), 264 restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"), 265 restic.DataBlob, 0, 25, 266 }, { 267 restic.TestParseID("9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae"), 268 restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"), 269 restic.TreeBlob, 38, 100, 270 }, { 271 restic.TestParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66"), 272 restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"), 273 restic.DataBlob, 150, 123, 274 }, 275 } 276 277 var exampleLookupTest = struct { 278 packID restic.ID 279 blobs map[restic.ID]restic.BlobType 280 }{ 281 restic.TestParseID("73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c"), 282 map[restic.ID]restic.BlobType{ 283 restic.TestParseID("3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce"): restic.DataBlob, 284 restic.TestParseID("9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae"): restic.TreeBlob, 285 restic.TestParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66"): restic.DataBlob, 286 }, 287 } 288 289 func TestIndexUnserialize(t *testing.T) { 290 oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")} 291 292 idx, err := repository.DecodeIndex(docExample) 293 rtest.OK(t, err) 294 295 for _, test := range exampleTests { 296 list, found := idx.Lookup(test.id, test.tpe) 297 rtest.Assert(t, found, "Expected to find blob id %v", test.id.Str()) 298 299 if len(list) != 1 { 300 t.Errorf("expected one result for blob %v, got %v: %v", test.id.Str(), len(list), list) 301 } 302 blob := list[0] 303 304 t.Logf("looking for blob %v/%v, got %v", test.tpe, test.id.Str(), blob) 305 306 rtest.Equals(t, test.packID, blob.PackID) 307 rtest.Equals(t, test.tpe, blob.Type) 308 rtest.Equals(t, test.offset, blob.Offset) 309 rtest.Equals(t, test.length, blob.Length) 310 } 311 312 rtest.Equals(t, oldIdx, idx.Supersedes()) 313 314 blobs := idx.ListPack(exampleLookupTest.packID) 315 if len(blobs) != len(exampleLookupTest.blobs) { 316 t.Fatalf("expected %d blobs in pack, got %d", len(exampleLookupTest.blobs), len(blobs)) 317 } 318 319 for _, blob := range blobs { 320 b, ok := exampleLookupTest.blobs[blob.ID] 321 if !ok { 322 t.Errorf("unexpected blob %v found", blob.ID.Str()) 323 } 324 if blob.Type != b { 325 t.Errorf("unexpected type for blob %v: want %v, got %v", blob.ID.Str(), b, blob.Type) 326 } 327 } 328 } 329 330 func BenchmarkDecodeIndex(b *testing.B) { 331 b.ResetTimer() 332 333 for i := 0; i < b.N; i++ { 334 _, err := repository.DecodeIndex(docExample) 335 rtest.OK(b, err) 336 } 337 } 338 339 func TestIndexUnserializeOld(t *testing.T) { 340 idx, err := repository.DecodeOldIndex(docOldExample) 341 rtest.OK(t, err) 342 343 for _, test := range exampleTests { 344 list, found := idx.Lookup(test.id, test.tpe) 345 rtest.Assert(t, found, "Expected to find blob id %v", test.id.Str()) 346 347 if len(list) != 1 { 348 t.Errorf("expected one result for blob %v, got %v: %v", test.id.Str(), len(list), list) 349 } 350 blob := list[0] 351 352 rtest.Equals(t, test.packID, blob.PackID) 353 rtest.Equals(t, test.tpe, blob.Type) 354 rtest.Equals(t, test.offset, blob.Offset) 355 rtest.Equals(t, test.length, blob.Length) 356 } 357 358 rtest.Equals(t, 0, len(idx.Supersedes())) 359 } 360 361 func TestIndexPacks(t *testing.T) { 362 idx := repository.NewIndex() 363 packs := restic.NewIDSet() 364 365 for i := 0; i < 20; i++ { 366 packID := restic.NewRandomID() 367 idx.Store(restic.PackedBlob{ 368 Blob: restic.Blob{ 369 Type: restic.DataBlob, 370 ID: restic.NewRandomID(), 371 Offset: 0, 372 Length: 23, 373 }, 374 PackID: packID, 375 }) 376 377 packs.Insert(packID) 378 } 379 380 idxPacks := idx.Packs() 381 rtest.Assert(t, packs.Equals(idxPacks), "packs in index do not match packs added to index") 382 } 383 384 const maxPackSize = 16 * 1024 * 1024 385 386 // This function generates a (insecure) random ID, similar to NewRandomID 387 func NewRandomTestID(rng *rand.Rand) restic.ID { 388 id := restic.ID{} 389 rng.Read(id[:]) 390 return id 391 } 392 393 func createRandomIndex(rng *rand.Rand) (idx *repository.Index, lookupID restic.ID) { 394 idx = repository.NewIndex() 395 396 // create index with 200k pack files 397 for i := 0; i < 200000; i++ { 398 packID := NewRandomTestID(rng) 399 offset := 0 400 for offset < maxPackSize { 401 size := 2000 + rand.Intn(4*1024*1024) 402 id := NewRandomTestID(rng) 403 idx.Store(restic.PackedBlob{ 404 PackID: packID, 405 Blob: restic.Blob{ 406 Type: restic.DataBlob, 407 ID: id, 408 Length: uint(size), 409 Offset: uint(offset), 410 }, 411 }) 412 413 offset += size 414 415 if rand.Float32() < 0.001 && lookupID.IsNull() { 416 lookupID = id 417 } 418 } 419 } 420 421 return idx, lookupID 422 } 423 424 func BenchmarkIndexHasUnknown(b *testing.B) { 425 idx, _ := createRandomIndex(rand.New(rand.NewSource(0))) 426 lookupID := restic.NewRandomID() 427 428 b.ResetTimer() 429 430 for i := 0; i < b.N; i++ { 431 idx.Has(lookupID, restic.DataBlob) 432 } 433 } 434 435 func BenchmarkIndexHasKnown(b *testing.B) { 436 idx, lookupID := createRandomIndex(rand.New(rand.NewSource(0))) 437 438 b.ResetTimer() 439 440 for i := 0; i < b.N; i++ { 441 idx.Has(lookupID, restic.DataBlob) 442 } 443 } 444 445 func TestIndexHas(t *testing.T) { 446 type testEntry struct { 447 id restic.ID 448 pack restic.ID 449 tpe restic.BlobType 450 offset, length uint 451 } 452 tests := []testEntry{} 453 454 idx := repository.NewIndex() 455 456 // create 50 packs with 20 blobs each 457 for i := 0; i < 50; i++ { 458 packID := restic.NewRandomID() 459 460 pos := uint(0) 461 for j := 0; j < 20; j++ { 462 id := restic.NewRandomID() 463 length := uint(i*100 + j) 464 idx.Store(restic.PackedBlob{ 465 Blob: restic.Blob{ 466 Type: restic.DataBlob, 467 ID: id, 468 Offset: pos, 469 Length: length, 470 }, 471 PackID: packID, 472 }) 473 474 tests = append(tests, testEntry{ 475 id: id, 476 pack: packID, 477 tpe: restic.DataBlob, 478 offset: pos, 479 length: length, 480 }) 481 482 pos += length 483 } 484 } 485 486 for _, testBlob := range tests { 487 rtest.Assert(t, idx.Has(testBlob.id, testBlob.tpe), "Index reports not having data blob added to it") 488 } 489 490 rtest.Assert(t, !idx.Has(restic.NewRandomID(), restic.DataBlob), "Index reports having a data blob not added to it") 491 rtest.Assert(t, !idx.Has(tests[0].id, restic.TreeBlob), "Index reports having a tree blob added to it with the same id as a data blob") 492 }