github.com/2lambda123/git-lfs@v2.5.2+incompatible/tq/tus_upload.go (about) 1 package tq 2 3 import ( 4 "fmt" 5 "io" 6 "io/ioutil" 7 "os" 8 "strconv" 9 "strings" 10 11 "github.com/git-lfs/git-lfs/errors" 12 "github.com/git-lfs/git-lfs/lfsapi" 13 "github.com/git-lfs/git-lfs/tools" 14 ) 15 16 const ( 17 TusAdapterName = "tus" 18 TusVersion = "1.0.0" 19 ) 20 21 // Adapter for tus.io protocol resumaable uploads 22 type tusUploadAdapter struct { 23 *adapterBase 24 } 25 26 func (a *tusUploadAdapter) ClearTempStorage() error { 27 // nothing to do, all temp state is on the server end 28 return nil 29 } 30 31 func (a *tusUploadAdapter) WorkerStarting(workerNum int) (interface{}, error) { 32 return nil, nil 33 } 34 func (a *tusUploadAdapter) WorkerEnding(workerNum int, ctx interface{}) { 35 } 36 37 func (a *tusUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb ProgressCallback, authOkFunc func()) error { 38 rel, err := t.Rel("upload") 39 if err != nil { 40 return err 41 } 42 if rel == nil { 43 return errors.Errorf("No upload action for object: %s", t.Oid) 44 } 45 46 // Note not supporting the Creation extension since the batch API generates URLs 47 // Also not supporting Concatenation to support parallel uploads of chunks; forward only 48 49 // 1. Send HEAD request to determine upload start point 50 // Request must include Tus-Resumable header (version) 51 a.Trace("xfer: sending tus.io HEAD request for %q", t.Oid) 52 req, err := a.newHTTPRequest("HEAD", rel) 53 if err != nil { 54 return err 55 } 56 57 req.Header.Set("Tus-Resumable", TusVersion) 58 59 res, err := a.doHTTP(t, req) 60 if err != nil { 61 return errors.NewRetriableError(err) 62 } 63 64 // Response will contain Upload-Offset if supported 65 offHdr := res.Header.Get("Upload-Offset") 66 if len(offHdr) == 0 { 67 return fmt.Errorf("Missing Upload-Offset header from tus.io HEAD response at %q, contact server admin", rel.Href) 68 } 69 offset, err := strconv.ParseInt(offHdr, 10, 64) 70 if err != nil || offset < 0 { 71 return fmt.Errorf("Invalid Upload-Offset value %q in response from tus.io HEAD at %q, contact server admin", offHdr, rel.Href) 72 } 73 // Upload-Offset=size means already completed (skip) 74 // Batch API will probably already detect this, but handle just in case 75 if offset >= t.Size { 76 a.Trace("xfer: tus.io HEAD offset %d indicates %q is already fully uploaded, skipping", offset, t.Oid) 77 advanceCallbackProgress(cb, t, t.Size) 78 return nil 79 } 80 81 // Open file for uploading 82 f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644) 83 if err != nil { 84 return errors.Wrap(err, "tus upload") 85 } 86 defer f.Close() 87 88 // Upload-Offset=0 means start from scratch, but still send PATCH 89 if offset == 0 { 90 a.Trace("xfer: tus.io uploading %q from start", t.Oid) 91 } else { 92 a.Trace("xfer: tus.io resuming upload %q from %d", t.Oid, offset) 93 advanceCallbackProgress(cb, t, offset) 94 } 95 96 // 2. Send PATCH request with byte start point (even if 0) in Upload-Offset 97 // Response status must be 204 98 // Response Upload-Offset must be request Upload-Offset plus sent bytes 99 // Response may include Upload-Expires header in which case check not passed 100 101 a.Trace("xfer: sending tus.io PATCH request for %q", t.Oid) 102 req, err = a.newHTTPRequest("PATCH", rel) 103 if err != nil { 104 return err 105 } 106 107 req.Header.Set("Tus-Resumable", TusVersion) 108 req.Header.Set("Upload-Offset", strconv.FormatInt(offset, 10)) 109 req.Header.Set("Content-Type", "application/offset+octet-stream") 110 req.Header.Set("Content-Length", strconv.FormatInt(t.Size-offset, 10)) 111 req.ContentLength = t.Size - offset 112 113 // Ensure progress callbacks made while uploading 114 // Wrap callback to give name context 115 ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error { 116 if cb != nil { 117 return cb(t.Name, totalSize, readSoFar, readSinceLast) 118 } 119 return nil 120 } 121 122 var reader lfsapi.ReadSeekCloser = tools.NewBodyWithCallback(f, t.Size, ccb) 123 reader = newStartCallbackReader(reader, func() error { 124 // seek to the offset since lfsapi.Client rewinds the body 125 if _, err := f.Seek(offset, os.SEEK_CUR); err != nil { 126 return err 127 } 128 // Signal auth was ok on first read; this frees up other workers to start 129 if authOkFunc != nil { 130 authOkFunc() 131 } 132 return nil 133 }) 134 135 req.Body = reader 136 137 req = a.apiClient.LogRequest(req, "lfs.data.upload") 138 res, err = a.doHTTP(t, req) 139 if err != nil { 140 return errors.NewRetriableError(err) 141 } 142 143 // A status code of 403 likely means that an authentication token for the 144 // upload has expired. This can be safely retried. 145 if res.StatusCode == 403 { 146 err = errors.New("http: received status 403") 147 return errors.NewRetriableError(err) 148 } 149 150 if res.StatusCode > 299 { 151 return errors.Wrapf(nil, "Invalid status for %s %s: %d", 152 req.Method, 153 strings.SplitN(req.URL.String(), "?", 2)[0], 154 res.StatusCode, 155 ) 156 } 157 158 io.Copy(ioutil.Discard, res.Body) 159 res.Body.Close() 160 161 return verifyUpload(a.apiClient, a.remote, t) 162 } 163 164 func configureTusAdapter(m *Manifest) { 165 m.RegisterNewAdapterFunc(TusAdapterName, Upload, func(name string, dir Direction) Adapter { 166 switch dir { 167 case Upload: 168 bu := &tusUploadAdapter{newAdapterBase(m.fs, name, dir, nil)} 169 // self implements impl 170 bu.transferImpl = bu 171 return bu 172 case Download: 173 panic("Should never ask tus.io to download") 174 } 175 return nil 176 }) 177 }