github.com/mckael/restic@v0.8.3/internal/pack/pack.go (about) 1 package pack 2 3 import ( 4 "bytes" 5 "encoding/binary" 6 "fmt" 7 "io" 8 "sync" 9 10 "github.com/restic/restic/internal/debug" 11 "github.com/restic/restic/internal/errors" 12 "github.com/restic/restic/internal/restic" 13 14 "github.com/restic/restic/internal/crypto" 15 ) 16 17 // Packer is used to create a new Pack. 18 type Packer struct { 19 blobs []restic.Blob 20 21 bytes uint 22 k *crypto.Key 23 wr io.Writer 24 25 m sync.Mutex 26 } 27 28 // NewPacker returns a new Packer that can be used to pack blobs 29 // together. If wr is nil, a bytes.Buffer is used. 30 func NewPacker(k *crypto.Key, wr io.Writer) *Packer { 31 if wr == nil { 32 wr = bytes.NewBuffer(nil) 33 } 34 return &Packer{k: k, wr: wr} 35 } 36 37 // Add saves the data read from rd as a new blob to the packer. Returned is the 38 // number of bytes written to the pack. 39 func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte) (int, error) { 40 p.m.Lock() 41 defer p.m.Unlock() 42 43 c := restic.Blob{Type: t, ID: id} 44 45 n, err := p.wr.Write(data) 46 c.Length = uint(n) 47 c.Offset = p.bytes 48 p.bytes += uint(n) 49 p.blobs = append(p.blobs, c) 50 51 return n, errors.Wrap(err, "Write") 52 } 53 54 var entrySize = uint(binary.Size(restic.BlobType(0)) + binary.Size(uint32(0)) + len(restic.ID{})) 55 56 // headerEntry is used with encoding/binary to read and write header entries 57 type headerEntry struct { 58 Type uint8 59 Length uint32 60 ID restic.ID 61 } 62 63 // Finalize writes the header for all added blobs and finalizes the pack. 64 // Returned are the number of bytes written, including the header. If the 65 // underlying writer implements io.Closer, it is closed. 66 func (p *Packer) Finalize() (uint, error) { 67 p.m.Lock() 68 defer p.m.Unlock() 69 70 bytesWritten := p.bytes 71 72 hdrBuf := bytes.NewBuffer(nil) 73 bytesHeader, err := p.writeHeader(hdrBuf) 74 if err != nil { 75 return 0, err 76 } 77 78 encryptedHeader := make([]byte, 0, hdrBuf.Len()+p.k.Overhead()+p.k.NonceSize()) 79 nonce := crypto.NewRandomNonce() 80 encryptedHeader = append(encryptedHeader, nonce...) 81 encryptedHeader = p.k.Seal(encryptedHeader, nonce, hdrBuf.Bytes(), nil) 82 83 // append the header 84 n, err := p.wr.Write(encryptedHeader) 85 if err != nil { 86 return 0, errors.Wrap(err, "Write") 87 } 88 89 hdrBytes := restic.CiphertextLength(int(bytesHeader)) 90 if n != hdrBytes { 91 return 0, errors.New("wrong number of bytes written") 92 } 93 94 bytesWritten += uint(hdrBytes) 95 96 // write length 97 err = binary.Write(p.wr, binary.LittleEndian, uint32(restic.CiphertextLength(len(p.blobs)*int(entrySize)))) 98 if err != nil { 99 return 0, errors.Wrap(err, "binary.Write") 100 } 101 bytesWritten += uint(binary.Size(uint32(0))) 102 103 p.bytes = uint(bytesWritten) 104 105 if w, ok := p.wr.(io.Closer); ok { 106 return bytesWritten, w.Close() 107 } 108 109 return bytesWritten, nil 110 } 111 112 // writeHeader constructs and writes the header to wr. 113 func (p *Packer) writeHeader(wr io.Writer) (bytesWritten uint, err error) { 114 for _, b := range p.blobs { 115 entry := headerEntry{ 116 Length: uint32(b.Length), 117 ID: b.ID, 118 } 119 120 switch b.Type { 121 case restic.DataBlob: 122 entry.Type = 0 123 case restic.TreeBlob: 124 entry.Type = 1 125 default: 126 return 0, errors.Errorf("invalid blob type %v", b.Type) 127 } 128 129 err := binary.Write(wr, binary.LittleEndian, entry) 130 if err != nil { 131 return bytesWritten, errors.Wrap(err, "binary.Write") 132 } 133 134 bytesWritten += entrySize 135 } 136 137 return 138 } 139 140 // Size returns the number of bytes written so far. 141 func (p *Packer) Size() uint { 142 p.m.Lock() 143 defer p.m.Unlock() 144 145 return p.bytes 146 } 147 148 // Count returns the number of blobs in this packer. 149 func (p *Packer) Count() int { 150 p.m.Lock() 151 defer p.m.Unlock() 152 153 return len(p.blobs) 154 } 155 156 // Blobs returns the slice of blobs that have been written. 157 func (p *Packer) Blobs() []restic.Blob { 158 p.m.Lock() 159 defer p.m.Unlock() 160 161 return p.blobs 162 } 163 164 // Writer return the underlying writer. 165 func (p *Packer) Writer() io.Writer { 166 return p.wr 167 } 168 169 func (p *Packer) String() string { 170 return fmt.Sprintf("<Packer %d blobs, %d bytes>", len(p.blobs), p.bytes) 171 } 172 173 var ( 174 // size of the header-length field at the end of the file 175 headerLengthSize = binary.Size(uint32(0)) 176 // we require at least one entry in the header, and one blob for a pack file 177 minFileSize = entrySize + crypto.Extension + uint(headerLengthSize) 178 ) 179 180 const ( 181 maxHeaderSize = 16 * 1024 * 1024 182 // number of header enries to download as part of header-length request 183 eagerEntries = 15 184 ) 185 186 // readRecords reads up to max records from the underlying ReaderAt, returning 187 // the raw header, the total number of records in the header, and any error. 188 // If the header contains fewer than max entries, the header is truncated to 189 // the appropriate size. 190 func readRecords(rd io.ReaderAt, size int64, max int) ([]byte, int, error) { 191 var bufsize int 192 bufsize += max * int(entrySize) 193 bufsize += crypto.Extension 194 bufsize += headerLengthSize 195 196 if bufsize > int(size) { 197 bufsize = int(size) 198 } 199 200 b := make([]byte, bufsize) 201 off := size - int64(bufsize) 202 if _, err := rd.ReadAt(b, off); err != nil { 203 return nil, 0, err 204 } 205 206 hlen := binary.LittleEndian.Uint32(b[len(b)-headerLengthSize:]) 207 b = b[:len(b)-headerLengthSize] 208 debug.Log("header length: %v", hlen) 209 210 var err error 211 switch { 212 case hlen == 0: 213 err = InvalidFileError{Message: "header length is zero"} 214 case hlen < crypto.Extension: 215 err = InvalidFileError{Message: "header length is too small"} 216 case (hlen-crypto.Extension)%uint32(entrySize) != 0: 217 err = InvalidFileError{Message: "header length is invalid"} 218 case int64(hlen) > size-int64(headerLengthSize): 219 err = InvalidFileError{Message: "header is larger than file"} 220 case int64(hlen) > maxHeaderSize: 221 err = InvalidFileError{Message: "header is larger than maxHeaderSize"} 222 } 223 if err != nil { 224 return nil, 0, errors.Wrap(err, "readHeader") 225 } 226 227 total := (int(hlen) - crypto.Extension) / int(entrySize) 228 if total < max { 229 // truncate to the beginning of the pack header 230 b = b[len(b)-int(hlen):] 231 } 232 233 return b, total, nil 234 } 235 236 // readHeader reads the header at the end of rd. size is the length of the 237 // whole data accessible in rd. 238 func readHeader(rd io.ReaderAt, size int64) ([]byte, error) { 239 debug.Log("size: %v", size) 240 if size < int64(minFileSize) { 241 err := InvalidFileError{Message: "file is too small"} 242 return nil, errors.Wrap(err, "readHeader") 243 } 244 245 // assuming extra request is significantly slower than extra bytes download, 246 // eagerly download eagerEntries header entries as part of header-length request. 247 // only make second request if actual number of entries is greater than eagerEntries 248 249 b, c, err := readRecords(rd, size, eagerEntries) 250 if err != nil { 251 return nil, err 252 } 253 if c <= eagerEntries { 254 // eager read sufficed, return what we got 255 return b, nil 256 } 257 b, _, err = readRecords(rd, size, c) 258 if err != nil { 259 return nil, err 260 } 261 return b, nil 262 } 263 264 // InvalidFileError is return when a file is found that is not a pack file. 265 type InvalidFileError struct { 266 Message string 267 } 268 269 func (e InvalidFileError) Error() string { 270 return e.Message 271 } 272 273 // List returns the list of entries found in a pack file. 274 func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, err error) { 275 buf, err := readHeader(rd, size) 276 if err != nil { 277 return nil, err 278 } 279 280 if len(buf) < k.NonceSize()+k.Overhead() { 281 return nil, errors.New("invalid header, too small") 282 } 283 284 nonce, buf := buf[:k.NonceSize()], buf[k.NonceSize():] 285 buf, err = k.Open(buf[:0], nonce, buf, nil) 286 if err != nil { 287 return nil, err 288 } 289 290 hdrRd := bytes.NewReader(buf) 291 292 entries = make([]restic.Blob, 0, uint(len(buf))/entrySize) 293 294 pos := uint(0) 295 for { 296 e := headerEntry{} 297 err = binary.Read(hdrRd, binary.LittleEndian, &e) 298 if errors.Cause(err) == io.EOF { 299 break 300 } 301 302 if err != nil { 303 return nil, errors.Wrap(err, "binary.Read") 304 } 305 306 entry := restic.Blob{ 307 Length: uint(e.Length), 308 ID: e.ID, 309 Offset: pos, 310 } 311 312 switch e.Type { 313 case 0: 314 entry.Type = restic.DataBlob 315 case 1: 316 entry.Type = restic.TreeBlob 317 default: 318 return nil, errors.Errorf("invalid type %d", e.Type) 319 } 320 321 entries = append(entries, entry) 322 323 pos += uint(e.Length) 324 } 325 326 return entries, nil 327 }