github.com/mgoltzsche/ctnr@v0.7.1-alpha/pkg/fs/source/sourceurl.go (about) 1 package source 2 3 import ( 4 "io" 5 "net/http" 6 "net/url" 7 "strconv" 8 9 "github.com/mgoltzsche/ctnr/pkg/fs" 10 "github.com/mgoltzsche/ctnr/pkg/idutils" 11 "github.com/pkg/errors" 12 ) 13 14 var _ fs.Source = &sourceURL{} 15 16 type sourceURL struct { 17 fs.FileAttrs 18 fs.DerivedAttrs 19 cache HttpHeaderCache 20 } 21 22 func NewSourceURL(url *url.URL, etagCache HttpHeaderCache, chown idutils.UserIds) fs.Source { 23 return &sourceURL{fs.FileAttrs{UserIds: chown, Mode: 0600}, fs.DerivedAttrs{URL: url.String()}, etagCache} 24 } 25 26 func (s *sourceURL) Attrs() fs.NodeInfo { 27 return fs.NodeInfo{fs.TypeFile, s.FileAttrs} 28 } 29 30 // Performs an HTTP cache validation request and/or obtains new header values. 31 // Either the cached or the new values are returned and can be used as cache key. 32 func (s *sourceURL) DeriveAttrs() (a fs.DerivedAttrs, err error) { 33 if s.HTTPInfo == "" { 34 defer func() { 35 if err != nil { 36 err = errors.Wrapf(err, "source URL %s", s.URL) 37 } 38 }() 39 var ( 40 cachedAttrs *HttpHeaders 41 req *http.Request 42 res *http.Response 43 client = &http.Client{} 44 ) 45 if cachedAttrs, err = s.cache.GetHttpHeaders(s.URL); err != nil { 46 return 47 } 48 if req, err = http.NewRequest(http.MethodGet, s.URL, nil); err != nil { 49 return 50 } 51 if cachedAttrs != nil { 52 if cachedAttrs.Etag != "" { 53 req.Header.Set("If-None-Match", cachedAttrs.Etag) 54 } 55 if cachedAttrs.LastModified != "" { 56 req.Header.Set("If-Modified-Since", cachedAttrs.LastModified) 57 } 58 } 59 if res, err = client.Do(req); err != nil { 60 return 61 } 62 defer func() { 63 if e := res.Body.Close(); e != nil && err == nil { 64 err = e 65 } 66 }() 67 if res.StatusCode == 304 && cachedAttrs != nil { 68 s.setUrlInfo(cachedAttrs) 69 } else if res.StatusCode == 200 { 70 urlInfo := urlInfo(res) 71 if err = s.cache.PutHttpHeaders(s.URL, urlInfo); err != nil { 72 return 73 } 74 s.setUrlInfo(urlInfo) 75 } else { 76 return a, errors.Errorf("returned HTTP code %d %s", res.StatusCode, res.Status) 77 } 78 } 79 return s.DerivedAttrs, nil 80 } 81 82 func urlInfo(r *http.Response) *HttpHeaders { 83 return &HttpHeaders{ 84 r.ContentLength, 85 r.Header.Get("ETag"), 86 r.Header.Get("Last-Modified"), 87 } 88 } 89 90 func (s *sourceURL) setUrlInfo(a *HttpHeaders) { 91 info := "" 92 if a.Etag != "" { 93 info = "etag:" + url.QueryEscape(a.Etag) 94 } 95 if a.LastModified != "" { 96 if info != "" { 97 info += "," 98 } 99 info += "time:" + url.QueryEscape(a.LastModified) 100 } else if a.ContentLength > 0 && a.Etag == "" { 101 info = "size=" + strconv.FormatInt(a.ContentLength, 10) 102 } 103 s.HTTPInfo = info 104 s.Size = a.ContentLength 105 } 106 107 func (s *sourceURL) Write(dest, name string, w fs.Writer, _ map[fs.Source]string) (err error) { 108 _, err = w.File(dest, s) 109 return errors.Wrap(err, "source URL") 110 } 111 112 func (s *sourceURL) Reader() (io.ReadCloser, error) { 113 res, err := http.Get(s.URL) 114 if err != nil { 115 return nil, err 116 } 117 if res.StatusCode != 200 { 118 res.Body.Close() 119 return nil, errors.Errorf("source URL %s: returned status %d %s", s.URL, res.StatusCode, res.Status) 120 } 121 // Size must be set here in order to stream URL into tar 122 s.Size = res.ContentLength 123 return res.Body, nil 124 } 125 126 func (s *sourceURL) HashIfAvailable() string { 127 return "" 128 } 129 130 func (s *sourceURL) Equal(o fs.Source) (bool, error) { 131 if s.Attrs().Equal(o.Attrs()) { 132 return false, nil 133 } 134 oa, err := o.DeriveAttrs() 135 if err != nil { 136 return false, errors.Wrap(err, "equal") 137 } 138 a, err := s.DeriveAttrs() 139 return a.Equal(&oa), errors.Wrap(err, "equal") 140 } 141 142 func (s *sourceURL) String() string { 143 return "sourceURL{" + s.URL + "}" 144 }