github.com/git-lfs/git-lfs@v2.5.2+incompatible/tq/basic_download.go (about) 1 package tq 2 3 import ( 4 "fmt" 5 "hash" 6 "io" 7 "os" 8 "path/filepath" 9 "regexp" 10 "strconv" 11 12 "github.com/git-lfs/git-lfs/errors" 13 "github.com/git-lfs/git-lfs/tools" 14 "github.com/rubyist/tracerx" 15 ) 16 17 // Adapter for basic HTTP downloads, includes resuming via HTTP Range 18 type basicDownloadAdapter struct { 19 *adapterBase 20 } 21 22 func (a *basicDownloadAdapter) ClearTempStorage() error { 23 return os.RemoveAll(a.tempDir()) 24 } 25 26 func (a *basicDownloadAdapter) tempDir() string { 27 // Must be dedicated to this adapter as deleted by ClearTempStorage 28 // Also make local to this repo not global, and separate to localstorage temp, 29 // which gets cleared at the end of every invocation 30 d := filepath.Join(a.fs.LFSStorageDir, "incomplete") 31 if err := os.MkdirAll(d, 0755); err != nil { 32 return os.TempDir() 33 } 34 return d 35 } 36 37 func (a *basicDownloadAdapter) WorkerStarting(workerNum int) (interface{}, error) { 38 return nil, nil 39 } 40 func (a *basicDownloadAdapter) WorkerEnding(workerNum int, ctx interface{}) { 41 } 42 43 func (a *basicDownloadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb ProgressCallback, authOkFunc func()) error { 44 f, fromByte, hashSoFar, err := a.checkResumeDownload(t) 45 if err != nil { 46 return err 47 } 48 return a.download(t, cb, authOkFunc, f, fromByte, hashSoFar) 49 } 50 51 // Checks to see if a download can be resumed, and if so returns a non-nil locked file, byte start and hash 52 func (a *basicDownloadAdapter) checkResumeDownload(t *Transfer) (outFile *os.File, fromByte int64, hashSoFar hash.Hash, e error) { 53 // lock the file by opening it for read/write, rather than checking Stat() etc 54 // which could be subject to race conditions by other processes 55 f, err := os.OpenFile(a.downloadFilename(t), os.O_RDWR, 0644) 56 57 if err != nil { 58 // Create a new file instead, must not already exist or error (permissions / race condition) 59 newfile, err := os.OpenFile(a.downloadFilename(t), os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0644) 60 return newfile, 0, nil, err 61 } 62 63 // Successfully opened an existing file at this point 64 // Read any existing data into hash then return file handle at end 65 hash := tools.NewLfsContentHash() 66 n, err := io.Copy(hash, f) 67 if err != nil { 68 f.Close() 69 return nil, 0, nil, err 70 } 71 tracerx.Printf("xfer: Attempting to resume download of %q from byte %d", t.Oid, n) 72 return f, n, hash, nil 73 74 } 75 76 // Create or open a download file for resuming 77 func (a *basicDownloadAdapter) downloadFilename(t *Transfer) string { 78 // Not a temp file since we will be resuming it 79 return filepath.Join(a.tempDir(), t.Oid+".tmp") 80 } 81 82 // download starts or resumes and download. Always closes dlFile if non-nil 83 func (a *basicDownloadAdapter) download(t *Transfer, cb ProgressCallback, authOkFunc func(), dlFile *os.File, fromByte int64, hash hash.Hash) error { 84 if dlFile != nil { 85 // ensure we always close dlFile. Note that this does not conflict with the 86 // early close below, as close is idempotent. 87 defer dlFile.Close() 88 } 89 90 rel, err := t.Rel("download") 91 if err != nil { 92 return err 93 } 94 if rel == nil { 95 return errors.Errorf("Object %s not found on the server.", t.Oid) 96 } 97 98 req, err := a.newHTTPRequest("GET", rel) 99 if err != nil { 100 return err 101 } 102 103 if fromByte > 0 { 104 if dlFile == nil || hash == nil { 105 return fmt.Errorf("Cannot restart %v from %d without a file & hash", t.Oid, fromByte) 106 } 107 // We could just use a start byte, but since we know the length be specific 108 req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", fromByte, t.Size-1)) 109 } 110 111 req = a.apiClient.LogRequest(req, "lfs.data.download") 112 res, err := a.doHTTP(t, req) 113 if err != nil { 114 // Special-case status code 416 () - fall back 115 if fromByte > 0 && dlFile != nil && (res != nil && res.StatusCode == 416) { 116 tracerx.Printf("xfer: server rejected resume download request for %q from byte %d; re-downloading from start", t.Oid, fromByte) 117 dlFile.Close() 118 os.Remove(dlFile.Name()) 119 return a.download(t, cb, authOkFunc, nil, 0, nil) 120 } 121 return errors.NewRetriableError(err) 122 } 123 124 defer res.Body.Close() 125 126 // Range request must return 206 & content range to confirm 127 if fromByte > 0 { 128 rangeRequestOk := false 129 var failReason string 130 // check 206 and Content-Range, fall back if either not as expected 131 if res.StatusCode == 206 { 132 // Probably a successful range request, check Content-Range 133 if rangeHdr := res.Header.Get("Content-Range"); rangeHdr != "" { 134 regex := regexp.MustCompile(`bytes (\d+)\-.*`) 135 match := regex.FindStringSubmatch(rangeHdr) 136 if match != nil && len(match) > 1 { 137 contentStart, _ := strconv.ParseInt(match[1], 10, 64) 138 if contentStart == fromByte { 139 rangeRequestOk = true 140 } else { 141 failReason = fmt.Sprintf("Content-Range start byte incorrect: %s expected %d", match[1], fromByte) 142 } 143 } else { 144 failReason = fmt.Sprintf("badly formatted Content-Range header: %q", rangeHdr) 145 } 146 } else { 147 failReason = "missing Content-Range header in response" 148 } 149 } else { 150 failReason = fmt.Sprintf("expected status code 206, received %d", res.StatusCode) 151 } 152 if rangeRequestOk { 153 tracerx.Printf("xfer: server accepted resume download request: %q from byte %d", t.Oid, fromByte) 154 advanceCallbackProgress(cb, t, fromByte) 155 } else { 156 // Abort resume, perform regular download 157 tracerx.Printf("xfer: failed to resume download for %q from byte %d: %s. Re-downloading from start", t.Oid, fromByte, failReason) 158 dlFile.Close() 159 os.Remove(dlFile.Name()) 160 if res.StatusCode == 200 { 161 // If status code was 200 then server just ignored Range header and 162 // sent everything. Don't re-request, use this one from byte 0 163 dlFile = nil 164 fromByte = 0 165 hash = nil 166 } else { 167 // re-request needed 168 return a.download(t, cb, authOkFunc, nil, 0, nil) 169 } 170 } 171 } 172 173 // Signal auth OK on success response, before starting download to free up 174 // other workers immediately 175 if authOkFunc != nil { 176 authOkFunc() 177 } 178 179 var hasher *tools.HashingReader 180 httpReader := tools.NewRetriableReader(res.Body) 181 182 if fromByte > 0 && hash != nil { 183 // pre-load hashing reader with previous content 184 hasher = tools.NewHashingReaderPreloadHash(httpReader, hash) 185 } else { 186 hasher = tools.NewHashingReader(httpReader) 187 } 188 189 if dlFile == nil { 190 // New file start 191 dlFile, err = os.OpenFile(a.downloadFilename(t), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) 192 if err != nil { 193 return err 194 } 195 defer dlFile.Close() 196 } 197 dlfilename := dlFile.Name() 198 // Wrap callback to give name context 199 ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error { 200 if cb != nil { 201 return cb(t.Name, totalSize, readSoFar+fromByte, readSinceLast) 202 } 203 return nil 204 } 205 written, err := tools.CopyWithCallback(dlFile, hasher, res.ContentLength, ccb) 206 if err != nil { 207 return errors.Wrapf(err, "cannot write data to tempfile %q", dlfilename) 208 } 209 if err := dlFile.Close(); err != nil { 210 return fmt.Errorf("can't close tempfile %q: %v", dlfilename, err) 211 } 212 213 if actual := hasher.Hash(); actual != t.Oid { 214 return fmt.Errorf("Expected OID %s, got %s after %d bytes written", t.Oid, actual, written) 215 } 216 217 return tools.RenameFileCopyPermissions(dlfilename, t.Path) 218 } 219 220 func configureBasicDownloadAdapter(m *Manifest) { 221 m.RegisterNewAdapterFunc(BasicAdapterName, Download, func(name string, dir Direction) Adapter { 222 switch dir { 223 case Download: 224 bd := &basicDownloadAdapter{newAdapterBase(m.fs, name, dir, nil)} 225 // self implements impl 226 bd.transferImpl = bd 227 return bd 228 case Upload: 229 panic("Should never ask this func to upload") 230 } 231 return nil 232 }) 233 }