github.com/graemephi/kahugo@v0.62.3-0.20211121071557-d78c0423784d/tpl/data/resources.go (about) 1 // Copyright 2016 The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package data 15 16 import ( 17 "bytes" 18 "io/ioutil" 19 "net/http" 20 "net/url" 21 "path/filepath" 22 "time" 23 24 "github.com/pkg/errors" 25 26 "github.com/gohugoio/hugo/cache/filecache" 27 28 "github.com/gohugoio/hugo/config" 29 "github.com/gohugoio/hugo/helpers" 30 "github.com/spf13/afero" 31 ) 32 33 var ( 34 resSleep = time.Second * 2 // if JSON decoding failed sleep for n seconds before retrying 35 resRetries = 1 // number of retries to load the JSON from URL 36 ) 37 38 // getRemote loads the content of a remote file. This method is thread safe. 39 func (ns *Namespace) getRemote(cache *filecache.Cache, unmarshal func([]byte) (bool, error), req *http.Request) error { 40 url := req.URL.String() 41 var headers bytes.Buffer 42 req.Header.Write(&headers) 43 id := helpers.MD5String(url + headers.String()) 44 var handled bool 45 var retry bool 46 47 _, b, err := cache.GetOrCreateBytes(id, func() ([]byte, error) { 48 var err error 49 handled = true 50 for i := 0; i <= resRetries; i++ { 51 ns.deps.Log.Infof("Downloading: %s ...", url) 52 var res *http.Response 53 res, err = ns.client.Do(req) 54 if err != nil { 55 return nil, err 56 } 57 58 var b []byte 59 b, err = ioutil.ReadAll(res.Body) 60 if err != nil { 61 return nil, err 62 } 63 res.Body.Close() 64 65 if isHTTPError(res) { 66 return nil, errors.Errorf("Failed to retrieve remote file: %s, body: %q", http.StatusText(res.StatusCode), b) 67 } 68 69 retry, err = unmarshal(b) 70 71 if err == nil { 72 // Return it so it can be cached. 73 return b, nil 74 } 75 76 if !retry { 77 return nil, err 78 } 79 80 ns.deps.Log.Infof("Cannot read remote resource %s: %s", url, err) 81 ns.deps.Log.Infof("Retry #%d for %s and sleeping for %s", i+1, url, resSleep) 82 time.Sleep(resSleep) 83 } 84 85 return nil, err 86 }) 87 88 if !handled { 89 // This is cached content and should be correct. 90 _, err = unmarshal(b) 91 } 92 93 return err 94 } 95 96 // getLocal loads the content of a local file 97 func getLocal(url string, fs afero.Fs, cfg config.Provider) ([]byte, error) { 98 filename := filepath.Join(cfg.GetString("workingDir"), url) 99 return afero.ReadFile(fs, filename) 100 } 101 102 // getResource loads the content of a local or remote file and returns its content and the 103 // cache ID used, if relevant. 104 func (ns *Namespace) getResource(cache *filecache.Cache, unmarshal func(b []byte) (bool, error), req *http.Request) error { 105 switch req.URL.Scheme { 106 case "": 107 url, err := url.QueryUnescape(req.URL.String()) 108 if err != nil { 109 return err 110 } 111 b, err := getLocal(url, ns.deps.Fs.Source, ns.deps.Cfg) 112 if err != nil { 113 return err 114 } 115 _, err = unmarshal(b) 116 return err 117 default: 118 return ns.getRemote(cache, unmarshal, req) 119 } 120 } 121 122 func isHTTPError(res *http.Response) bool { 123 return res.StatusCode < 200 || res.StatusCode > 299 124 }