github.com/lalkh/containerd@v1.4.3/content/helpers.go (about) 1 /* 2 Copyright The containerd Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package content 18 19 import ( 20 "context" 21 "io" 22 "io/ioutil" 23 "math/rand" 24 "sync" 25 "time" 26 27 "github.com/containerd/containerd/errdefs" 28 "github.com/opencontainers/go-digest" 29 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 30 "github.com/pkg/errors" 31 ) 32 33 var bufPool = sync.Pool{ 34 New: func() interface{} { 35 buffer := make([]byte, 1<<20) 36 return &buffer 37 }, 38 } 39 40 // NewReader returns a io.Reader from a ReaderAt 41 func NewReader(ra ReaderAt) io.Reader { 42 rd := io.NewSectionReader(ra, 0, ra.Size()) 43 return rd 44 } 45 46 // ReadBlob retrieves the entire contents of the blob from the provider. 47 // 48 // Avoid using this for large blobs, such as layers. 49 func ReadBlob(ctx context.Context, provider Provider, desc ocispec.Descriptor) ([]byte, error) { 50 ra, err := provider.ReaderAt(ctx, desc) 51 if err != nil { 52 return nil, err 53 } 54 defer ra.Close() 55 56 p := make([]byte, ra.Size()) 57 58 n, err := ra.ReadAt(p, 0) 59 if err == io.EOF { 60 if int64(n) != ra.Size() { 61 err = io.ErrUnexpectedEOF 62 } else { 63 err = nil 64 } 65 } 66 return p, err 67 } 68 69 // WriteBlob writes data with the expected digest into the content store. If 70 // expected already exists, the method returns immediately and the reader will 71 // not be consumed. 72 // 73 // This is useful when the digest and size are known beforehand. 74 // 75 // Copy is buffered, so no need to wrap reader in buffered io. 76 func WriteBlob(ctx context.Context, cs Ingester, ref string, r io.Reader, desc ocispec.Descriptor, opts ...Opt) error { 77 cw, err := OpenWriter(ctx, cs, WithRef(ref), WithDescriptor(desc)) 78 if err != nil { 79 if !errdefs.IsAlreadyExists(err) { 80 return errors.Wrap(err, "failed to open writer") 81 } 82 83 return nil // all ready present 84 } 85 defer cw.Close() 86 87 return Copy(ctx, cw, r, desc.Size, desc.Digest, opts...) 88 } 89 90 // OpenWriter opens a new writer for the given reference, retrying if the writer 91 // is locked until the reference is available or returns an error. 92 func OpenWriter(ctx context.Context, cs Ingester, opts ...WriterOpt) (Writer, error) { 93 var ( 94 cw Writer 95 err error 96 retry = 16 97 ) 98 for { 99 cw, err = cs.Writer(ctx, opts...) 100 if err != nil { 101 if !errdefs.IsUnavailable(err) { 102 return nil, err 103 } 104 105 // TODO: Check status to determine if the writer is active, 106 // continue waiting while active, otherwise return lock 107 // error or abort. Requires asserting for an ingest manager 108 109 select { 110 case <-time.After(time.Millisecond * time.Duration(rand.Intn(retry))): 111 if retry < 2048 { 112 retry = retry << 1 113 } 114 continue 115 case <-ctx.Done(): 116 // Propagate lock error 117 return nil, err 118 } 119 120 } 121 break 122 } 123 124 return cw, err 125 } 126 127 // Copy copies data with the expected digest from the reader into the 128 // provided content store writer. This copy commits the writer. 129 // 130 // This is useful when the digest and size are known beforehand. When 131 // the size or digest is unknown, these values may be empty. 132 // 133 // Copy is buffered, so no need to wrap reader in buffered io. 134 func Copy(ctx context.Context, cw Writer, r io.Reader, size int64, expected digest.Digest, opts ...Opt) error { 135 ws, err := cw.Status() 136 if err != nil { 137 return errors.Wrap(err, "failed to get status") 138 } 139 140 if ws.Offset > 0 { 141 r, err = seekReader(r, ws.Offset, size) 142 if err != nil { 143 return errors.Wrapf(err, "unable to resume write to %v", ws.Ref) 144 } 145 } 146 147 if _, err := copyWithBuffer(cw, r); err != nil { 148 return errors.Wrap(err, "failed to copy") 149 } 150 151 if err := cw.Commit(ctx, size, expected, opts...); err != nil { 152 if !errdefs.IsAlreadyExists(err) { 153 return errors.Wrapf(err, "failed commit on ref %q", ws.Ref) 154 } 155 } 156 157 return nil 158 } 159 160 // CopyReaderAt copies to a writer from a given reader at for the given 161 // number of bytes. This copy does not commit the writer. 162 func CopyReaderAt(cw Writer, ra ReaderAt, n int64) error { 163 ws, err := cw.Status() 164 if err != nil { 165 return err 166 } 167 168 _, err = copyWithBuffer(cw, io.NewSectionReader(ra, ws.Offset, n)) 169 return err 170 } 171 172 // CopyReader copies to a writer from a given reader, returning 173 // the number of bytes copied. 174 // Note: if the writer has a non-zero offset, the total number 175 // of bytes read may be greater than those copied if the reader 176 // is not an io.Seeker. 177 // This copy does not commit the writer. 178 func CopyReader(cw Writer, r io.Reader) (int64, error) { 179 ws, err := cw.Status() 180 if err != nil { 181 return 0, errors.Wrap(err, "failed to get status") 182 } 183 184 if ws.Offset > 0 { 185 r, err = seekReader(r, ws.Offset, 0) 186 if err != nil { 187 return 0, errors.Wrapf(err, "unable to resume write to %v", ws.Ref) 188 } 189 } 190 191 return copyWithBuffer(cw, r) 192 } 193 194 // seekReader attempts to seek the reader to the given offset, either by 195 // resolving `io.Seeker`, by detecting `io.ReaderAt`, or discarding 196 // up to the given offset. 197 func seekReader(r io.Reader, offset, size int64) (io.Reader, error) { 198 // attempt to resolve r as a seeker and setup the offset. 199 seeker, ok := r.(io.Seeker) 200 if ok { 201 nn, err := seeker.Seek(offset, io.SeekStart) 202 if nn != offset { 203 return nil, errors.Wrapf(err, "failed to seek to offset %v", offset) 204 } 205 206 if err != nil { 207 return nil, err 208 } 209 210 return r, nil 211 } 212 213 // ok, let's try io.ReaderAt! 214 readerAt, ok := r.(io.ReaderAt) 215 if ok && size > offset { 216 sr := io.NewSectionReader(readerAt, offset, size) 217 return sr, nil 218 } 219 220 // well then, let's just discard up to the offset 221 n, err := copyWithBuffer(ioutil.Discard, io.LimitReader(r, offset)) 222 if err != nil { 223 return nil, errors.Wrap(err, "failed to discard to offset") 224 } 225 if n != offset { 226 return nil, errors.Errorf("unable to discard to offset") 227 } 228 229 return r, nil 230 } 231 232 // copyWithBuffer is very similar to io.CopyBuffer https://golang.org/pkg/io/#CopyBuffer 233 // but instead of using Read to read from the src, we use ReadAtLeast to make sure we have 234 // a full buffer before we do a write operation to dst to reduce overheads associated 235 // with the write operations of small buffers. 236 func copyWithBuffer(dst io.Writer, src io.Reader) (written int64, err error) { 237 // If the reader has a WriteTo method, use it to do the copy. 238 // Avoids an allocation and a copy. 239 if wt, ok := src.(io.WriterTo); ok { 240 return wt.WriteTo(dst) 241 } 242 // Similarly, if the writer has a ReadFrom method, use it to do the copy. 243 if rt, ok := dst.(io.ReaderFrom); ok { 244 return rt.ReadFrom(src) 245 } 246 bufRef := bufPool.Get().(*[]byte) 247 defer bufPool.Put(bufRef) 248 buf := *bufRef 249 for { 250 nr, er := io.ReadAtLeast(src, buf, len(buf)) 251 if nr > 0 { 252 nw, ew := dst.Write(buf[0:nr]) 253 if nw > 0 { 254 written += int64(nw) 255 } 256 if ew != nil { 257 err = ew 258 break 259 } 260 if nr != nw { 261 err = io.ErrShortWrite 262 break 263 } 264 } 265 if er != nil { 266 // If an EOF happens after reading fewer than the requested bytes, 267 // ReadAtLeast returns ErrUnexpectedEOF. 268 if er != io.EOF && er != io.ErrUnexpectedEOF { 269 err = er 270 } 271 break 272 } 273 } 274 return 275 }