github.com/mckael/restic@v0.8.3/internal/repository/repository.go (about) 1 package repository 2 3 import ( 4 "bytes" 5 "context" 6 "crypto/sha256" 7 "encoding/json" 8 "fmt" 9 "io" 10 "os" 11 12 "github.com/restic/restic/internal/cache" 13 "github.com/restic/restic/internal/errors" 14 "github.com/restic/restic/internal/fs" 15 "github.com/restic/restic/internal/hashing" 16 "github.com/restic/restic/internal/restic" 17 18 "github.com/restic/restic/internal/backend" 19 "github.com/restic/restic/internal/crypto" 20 "github.com/restic/restic/internal/debug" 21 "github.com/restic/restic/internal/pack" 22 ) 23 24 // Repository is used to access a repository in a backend. 25 type Repository struct { 26 be restic.Backend 27 cfg restic.Config 28 key *crypto.Key 29 keyName string 30 idx *MasterIndex 31 restic.Cache 32 33 treePM *packerManager 34 dataPM *packerManager 35 } 36 37 // New returns a new repository with backend be. 38 func New(be restic.Backend) *Repository { 39 repo := &Repository{ 40 be: be, 41 idx: NewMasterIndex(), 42 dataPM: newPackerManager(be, nil), 43 treePM: newPackerManager(be, nil), 44 } 45 46 return repo 47 } 48 49 // Config returns the repository configuration. 50 func (r *Repository) Config() restic.Config { 51 return r.cfg 52 } 53 54 // UseCache replaces the backend with the wrapped cache. 55 func (r *Repository) UseCache(c restic.Cache) { 56 if c == nil { 57 return 58 } 59 debug.Log("using cache") 60 r.Cache = c 61 r.be = c.Wrap(r.be) 62 } 63 64 // PrefixLength returns the number of bytes required so that all prefixes of 65 // all IDs of type t are unique. 66 func (r *Repository) PrefixLength(t restic.FileType) (int, error) { 67 return restic.PrefixLength(r.be, t) 68 } 69 70 // LoadAndDecrypt loads and decrypts data identified by t and id from the 71 // backend. 72 func (r *Repository) LoadAndDecrypt(ctx context.Context, t restic.FileType, id restic.ID) (buf []byte, err error) { 73 debug.Log("load %v with id %v", t, id) 74 75 h := restic.Handle{Type: t, Name: id.String()} 76 buf, err = backend.LoadAll(ctx, r.be, h) 77 if err != nil { 78 debug.Log("error loading %v: %v", h, err) 79 return nil, err 80 } 81 82 if t != restic.ConfigFile && !restic.Hash(buf).Equal(id) { 83 return nil, errors.Errorf("load %v: invalid data returned", h) 84 } 85 86 nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] 87 plaintext, err := r.key.Open(ciphertext[:0], nonce, ciphertext, nil) 88 if err != nil { 89 return nil, err 90 } 91 92 return plaintext, nil 93 } 94 95 // sortCachedPacks moves all cached pack files to the front of blobs. 96 func (r *Repository) sortCachedPacks(blobs []restic.PackedBlob) []restic.PackedBlob { 97 if r.Cache == nil { 98 return blobs 99 } 100 101 cached := make([]restic.PackedBlob, 0, len(blobs)/2) 102 noncached := make([]restic.PackedBlob, 0, len(blobs)/2) 103 104 for _, blob := range blobs { 105 if r.Cache.Has(restic.Handle{Type: restic.DataFile, Name: blob.PackID.String()}) { 106 cached = append(cached, blob) 107 continue 108 } 109 noncached = append(noncached, blob) 110 } 111 112 return append(cached, noncached...) 113 } 114 115 // loadBlob tries to load and decrypt content identified by t and id from a 116 // pack from the backend, the result is stored in plaintextBuf, which must be 117 // large enough to hold the complete blob. 118 func (r *Repository) loadBlob(ctx context.Context, id restic.ID, t restic.BlobType, plaintextBuf []byte) (int, error) { 119 debug.Log("load %v with id %v (buf len %v, cap %d)", t, id, len(plaintextBuf), cap(plaintextBuf)) 120 121 // lookup packs 122 blobs, found := r.idx.Lookup(id, t) 123 if !found { 124 debug.Log("id %v not found in index", id) 125 return 0, errors.Errorf("id %v not found in repository", id) 126 } 127 128 // try cached pack files first 129 blobs = r.sortCachedPacks(blobs) 130 131 var lastError error 132 for _, blob := range blobs { 133 debug.Log("blob %v/%v found: %v", t, id, blob) 134 135 if blob.Type != t { 136 debug.Log("blob %v has wrong block type, want %v", blob, t) 137 } 138 139 // load blob from pack 140 h := restic.Handle{Type: restic.DataFile, Name: blob.PackID.String()} 141 142 if uint(cap(plaintextBuf)) < blob.Length { 143 return 0, errors.Errorf("buffer is too small: %v < %v", cap(plaintextBuf), blob.Length) 144 } 145 146 plaintextBuf = plaintextBuf[:blob.Length] 147 148 n, err := restic.ReadAt(ctx, r.be, h, int64(blob.Offset), plaintextBuf) 149 if err != nil { 150 debug.Log("error loading blob %v: %v", blob, err) 151 lastError = err 152 continue 153 } 154 155 if uint(n) != blob.Length { 156 lastError = errors.Errorf("error loading blob %v: wrong length returned, want %d, got %d", 157 id.Str(), blob.Length, uint(n)) 158 debug.Log("lastError: %v", lastError) 159 continue 160 } 161 162 // decrypt 163 nonce, ciphertext := plaintextBuf[:r.key.NonceSize()], plaintextBuf[r.key.NonceSize():] 164 plaintext, err := r.key.Open(ciphertext[:0], nonce, ciphertext, nil) 165 if err != nil { 166 lastError = errors.Errorf("decrypting blob %v failed: %v", id, err) 167 continue 168 } 169 170 // check hash 171 if !restic.Hash(plaintext).Equal(id) { 172 lastError = errors.Errorf("blob %v returned invalid hash", id) 173 continue 174 } 175 176 // move decrypted data to the start of the provided buffer 177 copy(plaintextBuf[0:], plaintext) 178 return len(plaintext), nil 179 } 180 181 if lastError != nil { 182 return 0, lastError 183 } 184 185 return 0, errors.Errorf("loading blob %v from %v packs failed", id.Str(), len(blobs)) 186 } 187 188 // LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on 189 // the item. 190 func (r *Repository) LoadJSONUnpacked(ctx context.Context, t restic.FileType, id restic.ID, item interface{}) (err error) { 191 buf, err := r.LoadAndDecrypt(ctx, t, id) 192 if err != nil { 193 return err 194 } 195 196 return json.Unmarshal(buf, item) 197 } 198 199 // LookupBlobSize returns the size of blob id. 200 func (r *Repository) LookupBlobSize(id restic.ID, tpe restic.BlobType) (uint, bool) { 201 return r.idx.LookupSize(id, tpe) 202 } 203 204 // SaveAndEncrypt encrypts data and stores it to the backend as type t. If data 205 // is small enough, it will be packed together with other small blobs. 206 func (r *Repository) SaveAndEncrypt(ctx context.Context, t restic.BlobType, data []byte, id *restic.ID) (restic.ID, error) { 207 if id == nil { 208 // compute plaintext hash 209 hashedID := restic.Hash(data) 210 id = &hashedID 211 } 212 213 debug.Log("save id %v (%v, %d bytes)", id, t, len(data)) 214 215 // get buf from the pool 216 ciphertext := getBuf() 217 defer freeBuf(ciphertext) 218 219 ciphertext = ciphertext[:0] 220 nonce := crypto.NewRandomNonce() 221 ciphertext = append(ciphertext, nonce...) 222 223 // encrypt blob 224 ciphertext = r.key.Seal(ciphertext, nonce, data, nil) 225 226 // find suitable packer and add blob 227 var pm *packerManager 228 229 switch t { 230 case restic.TreeBlob: 231 pm = r.treePM 232 case restic.DataBlob: 233 pm = r.dataPM 234 default: 235 panic(fmt.Sprintf("invalid type: %v", t)) 236 } 237 238 packer, err := pm.findPacker() 239 if err != nil { 240 return restic.ID{}, err 241 } 242 243 // save ciphertext 244 _, err = packer.Add(t, *id, ciphertext) 245 if err != nil { 246 return restic.ID{}, err 247 } 248 249 // if the pack is not full enough, put back to the list 250 if packer.Size() < minPackSize { 251 debug.Log("pack is not full enough (%d bytes)", packer.Size()) 252 pm.insertPacker(packer) 253 return *id, nil 254 } 255 256 // else write the pack to the backend 257 return *id, r.savePacker(ctx, t, packer) 258 } 259 260 // SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the 261 // backend as type t, without a pack. It returns the storage hash. 262 func (r *Repository) SaveJSONUnpacked(ctx context.Context, t restic.FileType, item interface{}) (restic.ID, error) { 263 debug.Log("save new blob %v", t) 264 plaintext, err := json.Marshal(item) 265 if err != nil { 266 return restic.ID{}, errors.Wrap(err, "json.Marshal") 267 } 268 269 return r.SaveUnpacked(ctx, t, plaintext) 270 } 271 272 // SaveUnpacked encrypts data and stores it in the backend. Returned is the 273 // storage hash. 274 func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []byte) (id restic.ID, err error) { 275 ciphertext := restic.NewBlobBuffer(len(p)) 276 ciphertext = ciphertext[:0] 277 nonce := crypto.NewRandomNonce() 278 ciphertext = append(ciphertext, nonce...) 279 280 ciphertext = r.key.Seal(ciphertext, nonce, p, nil) 281 282 id = restic.Hash(ciphertext) 283 h := restic.Handle{Type: t, Name: id.String()} 284 285 err = r.be.Save(ctx, h, bytes.NewReader(ciphertext)) 286 if err != nil { 287 debug.Log("error saving blob %v: %v", h, err) 288 return restic.ID{}, err 289 } 290 291 debug.Log("blob %v saved", h) 292 return id, nil 293 } 294 295 // Flush saves all remaining packs. 296 func (r *Repository) Flush(ctx context.Context) error { 297 pms := []struct { 298 t restic.BlobType 299 pm *packerManager 300 }{ 301 {restic.DataBlob, r.dataPM}, 302 {restic.TreeBlob, r.treePM}, 303 } 304 305 for _, p := range pms { 306 p.pm.pm.Lock() 307 308 debug.Log("manually flushing %d packs", len(p.pm.packers)) 309 for _, packer := range p.pm.packers { 310 err := r.savePacker(ctx, p.t, packer) 311 if err != nil { 312 p.pm.pm.Unlock() 313 return err 314 } 315 } 316 p.pm.packers = p.pm.packers[:0] 317 p.pm.pm.Unlock() 318 } 319 320 return nil 321 } 322 323 // Backend returns the backend for the repository. 324 func (r *Repository) Backend() restic.Backend { 325 return r.be 326 } 327 328 // Index returns the currently used MasterIndex. 329 func (r *Repository) Index() restic.Index { 330 return r.idx 331 } 332 333 // SetIndex instructs the repository to use the given index. 334 func (r *Repository) SetIndex(i restic.Index) { 335 r.idx = i.(*MasterIndex) 336 } 337 338 // SaveIndex saves an index in the repository. 339 func SaveIndex(ctx context.Context, repo restic.Repository, index *Index) (restic.ID, error) { 340 buf := bytes.NewBuffer(nil) 341 342 err := index.Finalize(buf) 343 if err != nil { 344 return restic.ID{}, err 345 } 346 347 return repo.SaveUnpacked(ctx, restic.IndexFile, buf.Bytes()) 348 } 349 350 // saveIndex saves all indexes in the backend. 351 func (r *Repository) saveIndex(ctx context.Context, indexes ...*Index) error { 352 for i, idx := range indexes { 353 debug.Log("Saving index %d", i) 354 355 sid, err := SaveIndex(ctx, r, idx) 356 if err != nil { 357 return err 358 } 359 360 debug.Log("Saved index %d as %v", i, sid) 361 } 362 363 return nil 364 } 365 366 // SaveIndex saves all new indexes in the backend. 367 func (r *Repository) SaveIndex(ctx context.Context) error { 368 return r.saveIndex(ctx, r.idx.NotFinalIndexes()...) 369 } 370 371 // SaveFullIndex saves all full indexes in the backend. 372 func (r *Repository) SaveFullIndex(ctx context.Context) error { 373 return r.saveIndex(ctx, r.idx.FullIndexes()...) 374 } 375 376 const loadIndexParallelism = 4 377 378 // LoadIndex loads all index files from the backend in parallel and stores them 379 // in the master index. The first error that occurred is returned. 380 func (r *Repository) LoadIndex(ctx context.Context) error { 381 debug.Log("Loading index") 382 383 errCh := make(chan error, 1) 384 indexes := make(chan *Index) 385 386 worker := func(ctx context.Context, id restic.ID) error { 387 idx, err := LoadIndex(ctx, r, id) 388 if err != nil { 389 fmt.Fprintf(os.Stderr, "%v, ignoring\n", err) 390 return nil 391 } 392 393 select { 394 case indexes <- idx: 395 case <-ctx.Done(): 396 } 397 398 return nil 399 } 400 401 go func() { 402 defer close(indexes) 403 errCh <- FilesInParallel(ctx, r.be, restic.IndexFile, loadIndexParallelism, 404 ParallelWorkFuncParseID(worker)) 405 }() 406 407 validIndex := restic.NewIDSet() 408 for idx := range indexes { 409 id, err := idx.ID() 410 if err == nil { 411 validIndex.Insert(id) 412 } 413 r.idx.Insert(idx) 414 } 415 416 if r.Cache != nil { 417 // clear old index files 418 err := r.Cache.Clear(restic.IndexFile, validIndex) 419 if err != nil { 420 fmt.Fprintf(os.Stderr, "error clearing index files in cache: %v\n", err) 421 } 422 423 packs := restic.NewIDSet() 424 for _, idx := range r.idx.All() { 425 for id := range idx.Packs() { 426 packs.Insert(id) 427 } 428 } 429 430 // clear old data files 431 err = r.Cache.Clear(restic.DataFile, packs) 432 if err != nil { 433 fmt.Fprintf(os.Stderr, "error clearing data files in cache: %v\n", err) 434 } 435 436 treePacks := restic.NewIDSet() 437 for _, idx := range r.idx.All() { 438 for _, id := range idx.TreePacks() { 439 treePacks.Insert(id) 440 } 441 } 442 443 // use readahead 444 cache := r.Cache.(*cache.Cache) 445 cache.PerformReadahead = func(h restic.Handle) bool { 446 if h.Type != restic.DataFile { 447 return false 448 } 449 450 id, err := restic.ParseID(h.Name) 451 if err != nil { 452 return false 453 } 454 455 return treePacks.Has(id) 456 } 457 } 458 459 if err := <-errCh; err != nil { 460 return err 461 } 462 463 return nil 464 } 465 466 // LoadIndex loads the index id from backend and returns it. 467 func LoadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*Index, error) { 468 idx, err := LoadIndexWithDecoder(ctx, repo, id, DecodeIndex) 469 if err == nil { 470 return idx, nil 471 } 472 473 if errors.Cause(err) == ErrOldIndexFormat { 474 fmt.Fprintf(os.Stderr, "index %v has old format\n", id.Str()) 475 return LoadIndexWithDecoder(ctx, repo, id, DecodeOldIndex) 476 } 477 478 return nil, err 479 } 480 481 // SearchKey finds a key with the supplied password, afterwards the config is 482 // read and parsed. It tries at most maxKeys key files in the repo. 483 func (r *Repository) SearchKey(ctx context.Context, password string, maxKeys int) error { 484 key, err := SearchKey(ctx, r, password, maxKeys) 485 if err != nil { 486 return err 487 } 488 489 r.key = key.master 490 r.dataPM.key = key.master 491 r.treePM.key = key.master 492 r.keyName = key.Name() 493 r.cfg, err = restic.LoadConfig(ctx, r) 494 return err 495 } 496 497 // Init creates a new master key with the supplied password, initializes and 498 // saves the repository config. 499 func (r *Repository) Init(ctx context.Context, password string) error { 500 has, err := r.be.Test(ctx, restic.Handle{Type: restic.ConfigFile}) 501 if err != nil { 502 return err 503 } 504 if has { 505 return errors.New("repository master key and config already initialized") 506 } 507 508 cfg, err := restic.CreateConfig() 509 if err != nil { 510 return err 511 } 512 513 return r.init(ctx, password, cfg) 514 } 515 516 // init creates a new master key with the supplied password and uses it to save 517 // the config into the repo. 518 func (r *Repository) init(ctx context.Context, password string, cfg restic.Config) error { 519 key, err := createMasterKey(r, password) 520 if err != nil { 521 return err 522 } 523 524 r.key = key.master 525 r.dataPM.key = key.master 526 r.treePM.key = key.master 527 r.keyName = key.Name() 528 r.cfg = cfg 529 _, err = r.SaveJSONUnpacked(ctx, restic.ConfigFile, cfg) 530 return err 531 } 532 533 // Key returns the current master key. 534 func (r *Repository) Key() *crypto.Key { 535 return r.key 536 } 537 538 // KeyName returns the name of the current key in the backend. 539 func (r *Repository) KeyName() string { 540 return r.keyName 541 } 542 543 // List runs fn for all files of type t in the repo. 544 func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error { 545 return r.be.List(ctx, t, func(fi restic.FileInfo) error { 546 id, err := restic.ParseID(fi.Name) 547 if err != nil { 548 debug.Log("unable to parse %v as an ID", fi.Name) 549 return nil 550 } 551 return fn(id, fi.Size) 552 }) 553 } 554 555 // ListPack returns the list of blobs saved in the pack id and the length of 556 // the file as stored in the backend. 557 func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) ([]restic.Blob, int64, error) { 558 h := restic.Handle{Type: restic.DataFile, Name: id.String()} 559 560 blobs, err := pack.List(r.Key(), restic.ReaderAt(r.Backend(), h), size) 561 if err != nil { 562 return nil, 0, err 563 } 564 565 return blobs, size, nil 566 } 567 568 // Delete calls backend.Delete() if implemented, and returns an error 569 // otherwise. 570 func (r *Repository) Delete(ctx context.Context) error { 571 return r.be.Delete(ctx) 572 } 573 574 // Close closes the repository by closing the backend. 575 func (r *Repository) Close() error { 576 return r.be.Close() 577 } 578 579 // LoadBlob loads a blob of type t from the repository to the buffer. buf must 580 // be large enough to hold the encrypted blob, since it is used as scratch 581 // space. 582 func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.ID, buf []byte) (int, error) { 583 debug.Log("load blob %v into buf (len %v, cap %v)", id, len(buf), cap(buf)) 584 size, found := r.idx.LookupSize(id, t) 585 if !found { 586 return 0, errors.Errorf("id %v not found in repository", id) 587 } 588 589 if cap(buf) < restic.CiphertextLength(int(size)) { 590 return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", cap(buf), restic.CiphertextLength(int(size))) 591 } 592 593 n, err := r.loadBlob(ctx, id, t, buf) 594 if err != nil { 595 return 0, err 596 } 597 buf = buf[:n] 598 599 debug.Log("loaded %d bytes into buf %p", len(buf), buf) 600 601 return len(buf), err 602 } 603 604 // SaveBlob saves a blob of type t into the repository. If id is the null id, it 605 // will be computed and returned. 606 func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID) (restic.ID, error) { 607 var i *restic.ID 608 if !id.IsNull() { 609 i = &id 610 } 611 return r.SaveAndEncrypt(ctx, t, buf, i) 612 } 613 614 // LoadTree loads a tree from the repository. 615 func (r *Repository) LoadTree(ctx context.Context, id restic.ID) (*restic.Tree, error) { 616 debug.Log("load tree %v", id) 617 618 size, found := r.idx.LookupSize(id, restic.TreeBlob) 619 if !found { 620 return nil, errors.Errorf("tree %v not found in repository", id) 621 } 622 623 debug.Log("size is %d, create buffer", size) 624 buf := restic.NewBlobBuffer(int(size)) 625 626 n, err := r.loadBlob(ctx, id, restic.TreeBlob, buf) 627 if err != nil { 628 return nil, err 629 } 630 buf = buf[:n] 631 632 t := &restic.Tree{} 633 err = json.Unmarshal(buf, t) 634 if err != nil { 635 return nil, err 636 } 637 638 return t, nil 639 } 640 641 // SaveTree stores a tree into the repository and returns the ID. The ID is 642 // checked against the index. The tree is only stored when the index does not 643 // contain the ID. 644 func (r *Repository) SaveTree(ctx context.Context, t *restic.Tree) (restic.ID, error) { 645 buf, err := json.Marshal(t) 646 if err != nil { 647 return restic.ID{}, errors.Wrap(err, "MarshalJSON") 648 } 649 650 // append a newline so that the data is always consistent (json.Encoder 651 // adds a newline after each object) 652 buf = append(buf, '\n') 653 654 id := restic.Hash(buf) 655 if r.idx.Has(id, restic.TreeBlob) { 656 return id, nil 657 } 658 659 _, err = r.SaveBlob(ctx, restic.TreeBlob, buf, id) 660 return id, err 661 } 662 663 // DownloadAndHash is all-in-one helper to download content of the file at h to a temporary filesystem location 664 // and calculate ID of the contents. Returned (temporary) file is positioned at the beginning of the file; 665 // it is reponsibility of the caller to close and delete the file. 666 func DownloadAndHash(ctx context.Context, repo restic.Repository, h restic.Handle) (tmpfile *os.File, hash restic.ID, size int64, err error) { 667 tmpfile, err = fs.TempFile("", "restic-temp-") 668 if err != nil { 669 return nil, restic.ID{}, -1, errors.Wrap(err, "TempFile") 670 } 671 672 err = repo.Backend().Load(ctx, h, 0, 0, func(rd io.Reader) (ierr error) { 673 _, ierr = tmpfile.Seek(0, io.SeekStart) 674 if ierr == nil { 675 ierr = tmpfile.Truncate(0) 676 } 677 if ierr != nil { 678 return ierr 679 } 680 hrd := hashing.NewReader(rd, sha256.New()) 681 size, ierr = io.Copy(tmpfile, hrd) 682 hash = restic.IDFromHash(hrd.Sum(nil)) 683 return ierr 684 }) 685 686 _, err = tmpfile.Seek(0, io.SeekStart) 687 if err != nil { 688 tmpfile.Close() 689 os.Remove(tmpfile.Name()) 690 return nil, restic.ID{}, -1, errors.Wrap(err, "Seek") 691 } 692 693 return tmpfile, hash, size, err 694 }