github.com/q2/git-lfs@v0.5.1-0.20150410234700-03a0d4cec40e/lfs/client.go (about) 1 package lfs 2 3 import ( 4 "bytes" 5 "encoding/base64" 6 "encoding/json" 7 "errors" 8 "fmt" 9 "github.com/cheggaaa/pb" 10 "github.com/rubyist/tracerx" 11 "io" 12 "io/ioutil" 13 "net/http" 14 "os" 15 "path/filepath" 16 "regexp" 17 "strconv" 18 ) 19 20 const ( 21 mediaType = "application/vnd.git-lfs+json; charset-utf-8" 22 ) 23 24 var ( 25 lfsMediaTypeRE = regexp.MustCompile(`\Aapplication/vnd\.git\-lfs\+json(;|\z)`) 26 jsonMediaTypeRE = regexp.MustCompile(`\Aapplication/json(;|\z)`) 27 objectRelationDoesNotExist = errors.New("relation does not exist") 28 hiddenHeaders = map[string]bool{ 29 "Authorization": true, 30 } 31 32 defaultErrors = map[int]string{ 33 400: "Client error: %s", 34 401: "Authorization error: %s\nCheck that you have proper access to the repository", 35 403: "Authorization error: %s\nCheck that you have proper access to the repository", 36 404: "Repository or object not found: %s\nCheck that it exists and that you have proper access to it", 37 500: "Server error: %s", 38 } 39 ) 40 41 type objectResource struct { 42 Oid string `json:"oid,omitempty"` 43 Size int64 `json:"size,omitempty"` 44 Links map[string]*linkRelation `json:"_links,omitempty"` 45 } 46 47 func (o *objectResource) NewRequest(relation, method string) (*http.Request, Creds, error) { 48 rel, ok := o.Rel(relation) 49 if !ok { 50 return nil, nil, objectRelationDoesNotExist 51 } 52 53 req, creds, err := newClientRequest(method, rel.Href) 54 if err != nil { 55 return nil, nil, err 56 } 57 58 for h, v := range rel.Header { 59 req.Header.Set(h, v) 60 } 61 62 return req, creds, nil 63 } 64 65 func (o *objectResource) Rel(name string) (*linkRelation, bool) { 66 if o.Links == nil { 67 return nil, false 68 } 69 70 rel, ok := o.Links[name] 71 return rel, ok 72 } 73 74 type linkRelation struct { 75 Href string `json:"href"` 76 Header map[string]string `json:"header,omitempty"` 77 } 78 79 type ClientError struct { 80 Message string `json:"message"` 81 DocumentationUrl string `json:"documentation_url,omitempty"` 82 RequestId string `json:"request_id,omitempty"` 83 } 84 85 func (e *ClientError) Error() string { 86 msg := e.Message 87 if len(e.DocumentationUrl) > 0 { 88 msg += "\nDocs: " + e.DocumentationUrl 89 } 90 if len(e.RequestId) > 0 { 91 msg += "\nRequest ID: " + e.RequestId 92 } 93 return msg 94 } 95 96 func Download(oid string) (io.ReadCloser, int64, *WrappedError) { 97 req, creds, err := newApiRequest("GET", oid) 98 if err != nil { 99 return nil, 0, Error(err) 100 } 101 102 res, obj, wErr := doApiRequest(req, creds) 103 if wErr != nil { 104 return nil, 0, wErr 105 } 106 107 req, creds, err = obj.NewRequest("download", "GET") 108 if err != nil { 109 return nil, 0, Error(err) 110 } 111 112 res, wErr = doHttpRequest(req, creds) 113 if wErr != nil { 114 return nil, 0, wErr 115 } 116 117 return res.Body, res.ContentLength, nil 118 } 119 120 type byteCloser struct { 121 *bytes.Reader 122 } 123 124 func (b *byteCloser) Close() error { 125 return nil 126 } 127 128 func Upload(oidPath, filename string, cb CopyCallback) *WrappedError { 129 oid := filepath.Base(oidPath) 130 file, err := os.Open(oidPath) 131 if err != nil { 132 return Error(err) 133 } 134 defer file.Close() 135 136 stat, err := file.Stat() 137 if err != nil { 138 return Error(err) 139 } 140 141 reqObj := &objectResource{ 142 Oid: oid, 143 Size: stat.Size(), 144 } 145 146 by, err := json.Marshal(reqObj) 147 if err != nil { 148 return Error(err) 149 } 150 151 req, creds, err := newApiRequest("POST", oid) 152 if err != nil { 153 return Error(err) 154 } 155 156 req.Header.Set("Content-Type", mediaType) 157 req.Header.Set("Content-Length", strconv.Itoa(len(by))) 158 req.ContentLength = int64(len(by)) 159 req.Body = &byteCloser{bytes.NewReader(by)} 160 161 tracerx.Printf("api: uploading %s (%s)", filename, oid) 162 res, obj, wErr := doApiRequest(req, creds) 163 if wErr != nil { 164 return wErr 165 } 166 167 if res.StatusCode == 200 { 168 return nil 169 } 170 171 req, creds, err = obj.NewRequest("upload", "PUT") 172 if err != nil { 173 return Error(err) 174 } 175 176 if len(req.Header.Get("Content-Type")) == 0 { 177 req.Header.Set("Content-Type", "application/octet-stream") 178 } 179 req.Header.Set("Content-Length", strconv.FormatInt(reqObj.Size, 10)) 180 req.ContentLength = reqObj.Size 181 182 reader := &CallbackReader{ 183 C: cb, 184 TotalSize: reqObj.Size, 185 Reader: file, 186 } 187 188 bar := pb.New64(reqObj.Size) 189 bar.SetUnits(pb.U_BYTES) 190 bar.Start() 191 192 req.Body = ioutil.NopCloser(bar.NewProxyReader(reader)) 193 194 res, wErr = doHttpRequest(req, creds) 195 if wErr != nil { 196 return wErr 197 } 198 199 if res.StatusCode > 299 { 200 return Errorf(nil, "Invalid status for %s %s: %d", req.Method, req.URL, res.StatusCode) 201 } 202 203 io.Copy(ioutil.Discard, res.Body) 204 res.Body.Close() 205 206 req, creds, err = obj.NewRequest("verify", "POST") 207 if err == objectRelationDoesNotExist { 208 return nil 209 } else if err != nil { 210 return Error(err) 211 } 212 213 req.Header.Set("Content-Type", mediaType) 214 req.Header.Set("Content-Length", strconv.Itoa(len(by))) 215 req.ContentLength = int64(len(by)) 216 req.Body = ioutil.NopCloser(bytes.NewReader(by)) 217 res, wErr = doHttpRequest(req, creds) 218 219 io.Copy(ioutil.Discard, res.Body) 220 res.Body.Close() 221 222 return wErr 223 } 224 225 func doHttpRequest(req *http.Request, creds Creds) (*http.Response, *WrappedError) { 226 res, err := DoHTTP(Config, req) 227 228 var wErr *WrappedError 229 230 if err != nil { 231 wErr = Errorf(err, "Error for %s %s", res.Request.Method, res.Request.URL) 232 } else { 233 if creds != nil { 234 saveCredentials(creds, res) 235 } 236 237 wErr = handleResponse(res) 238 } 239 240 if wErr != nil { 241 if res != nil { 242 setErrorResponseContext(wErr, res) 243 } else { 244 setErrorRequestContext(wErr, req) 245 } 246 } 247 248 return res, wErr 249 } 250 251 func doApiRequestWithRedirects(req *http.Request, creds Creds, via []*http.Request) (*http.Response, *WrappedError) { 252 res, wErr := doHttpRequest(req, creds) 253 if wErr != nil { 254 return res, wErr 255 } 256 257 if res.StatusCode == 307 { 258 redirectedReq, redirectedCreds, err := newClientRequest(req.Method, res.Header.Get("Location")) 259 if err != nil { 260 return res, Errorf(err, err.Error()) 261 } 262 263 via = append(via, req) 264 if seeker, ok := req.Body.(io.Seeker); ok { 265 _, err := seeker.Seek(0, 0) 266 if err != nil { 267 return res, Error(err) 268 } 269 redirectedReq.Body = req.Body 270 redirectedReq.ContentLength = req.ContentLength 271 } else { 272 return res, Errorf(nil, "Request body needs to be an io.Seeker to handle redirects.") 273 } 274 275 if err = checkRedirect(redirectedReq, via); err != nil { 276 return res, Errorf(err, err.Error()) 277 } 278 279 return doApiRequestWithRedirects(redirectedReq, redirectedCreds, via) 280 } 281 282 return res, wErr 283 } 284 285 func doApiRequest(req *http.Request, creds Creds) (*http.Response, *objectResource, *WrappedError) { 286 via := make([]*http.Request, 0, 4) 287 res, wErr := doApiRequestWithRedirects(req, creds, via) 288 if wErr != nil { 289 return res, nil, wErr 290 } 291 292 obj := &objectResource{} 293 wErr = decodeApiResponse(res, obj) 294 295 if wErr != nil { 296 setErrorResponseContext(wErr, res) 297 } 298 299 return res, obj, wErr 300 } 301 302 func handleResponse(res *http.Response) *WrappedError { 303 if res.StatusCode < 400 { 304 return nil 305 } 306 307 defer func() { 308 io.Copy(ioutil.Discard, res.Body) 309 res.Body.Close() 310 }() 311 312 cliErr := &ClientError{} 313 wErr := decodeApiResponse(res, cliErr) 314 if wErr == nil { 315 if len(cliErr.Message) == 0 { 316 wErr = defaultError(res) 317 } else { 318 wErr = Error(cliErr) 319 } 320 } 321 322 wErr.Panic = res.StatusCode > 499 && res.StatusCode != 501 && res.StatusCode != 509 323 return wErr 324 } 325 326 func decodeApiResponse(res *http.Response, obj interface{}) *WrappedError { 327 ctype := res.Header.Get("Content-Type") 328 if !(lfsMediaTypeRE.MatchString(ctype) || jsonMediaTypeRE.MatchString(ctype)) { 329 return nil 330 } 331 332 err := json.NewDecoder(res.Body).Decode(obj) 333 io.Copy(ioutil.Discard, res.Body) 334 res.Body.Close() 335 336 if err != nil { 337 return Errorf(err, "Unable to parse HTTP response for %s %s", res.Request.Method, res.Request.URL) 338 } 339 340 return nil 341 } 342 343 func defaultError(res *http.Response) *WrappedError { 344 var msgFmt string 345 346 if f, ok := defaultErrors[res.StatusCode]; ok { 347 msgFmt = f 348 } else if res.StatusCode < 500 { 349 msgFmt = defaultErrors[400] + fmt.Sprintf(" from HTTP %d", res.StatusCode) 350 } else { 351 msgFmt = defaultErrors[500] + fmt.Sprintf(" from HTTP %d", res.StatusCode) 352 } 353 354 return Error(fmt.Errorf(msgFmt, res.Request.URL)) 355 } 356 357 func saveCredentials(creds Creds, res *http.Response) { 358 if creds == nil { 359 return 360 } 361 362 if res.StatusCode < 300 { 363 execCreds(creds, "approve") 364 } else if res.StatusCode == 401 { 365 execCreds(creds, "reject") 366 } 367 } 368 369 func newApiRequest(method, oid string) (*http.Request, Creds, error) { 370 endpoint := Config.Endpoint() 371 objectOid := oid 372 operation := "download" 373 if method == "POST" { 374 objectOid = "" 375 operation = "upload" 376 } 377 378 u, err := ObjectUrl(endpoint, objectOid) 379 if err != nil { 380 return nil, nil, err 381 } 382 383 req, creds, err := newClientRequest(method, u.String()) 384 if err == nil { 385 req.Header.Set("Accept", mediaType) 386 if err := mergeSshHeader(req.Header, endpoint, operation, oid); err != nil { 387 tracerx.Printf("ssh: attempted with %s. Error: %s", 388 endpoint.SshUserAndHost, err.Error(), 389 ) 390 } 391 } 392 return req, creds, err 393 } 394 395 func newClientRequest(method, rawurl string) (*http.Request, Creds, error) { 396 req, err := http.NewRequest(method, rawurl, nil) 397 if err != nil { 398 return req, nil, err 399 } 400 401 req.Header.Set("User-Agent", UserAgent) 402 creds, err := getCreds(req) 403 return req, creds, err 404 } 405 406 func getCreds(req *http.Request) (Creds, error) { 407 if len(req.Header.Get("Authorization")) > 0 { 408 return nil, nil 409 } 410 411 apiUrl, err := Config.ObjectUrl("") 412 if err != nil { 413 return nil, err 414 } 415 416 if req.URL.Scheme == apiUrl.Scheme && 417 req.URL.Host == apiUrl.Host { 418 creds, err := credentials(req.URL) 419 if err != nil { 420 return nil, err 421 } 422 423 token := fmt.Sprintf("%s:%s", creds["username"], creds["password"]) 424 auth := "Basic " + base64.URLEncoding.EncodeToString([]byte(token)) 425 req.Header.Set("Authorization", auth) 426 return creds, nil 427 } 428 429 return nil, nil 430 } 431 432 func setErrorRequestContext(err *WrappedError, req *http.Request) { 433 err.Set("Endpoint", Config.Endpoint().Url) 434 err.Set("URL", fmt.Sprintf("%s %s", req.Method, req.URL.String())) 435 setErrorHeaderContext(err, "Response", req.Header) 436 } 437 438 func setErrorResponseContext(err *WrappedError, res *http.Response) { 439 err.Set("Status", res.Status) 440 setErrorHeaderContext(err, "Request", res.Header) 441 setErrorRequestContext(err, res.Request) 442 } 443 444 func setErrorHeaderContext(err *WrappedError, prefix string, head http.Header) { 445 for key, _ := range head { 446 contextKey := fmt.Sprintf("%s:%s", prefix, key) 447 if _, skip := hiddenHeaders[key]; skip { 448 err.Set(contextKey, "--") 449 } else { 450 err.Set(contextKey, head.Get(key)) 451 } 452 } 453 }