github.com/graemephi/kahugo@v0.62.3-0.20211121071557-d78c0423784d/tpl/data/resources.go (about)

     1  // Copyright 2016 The Hugo Authors. All rights reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  // http://www.apache.org/licenses/LICENSE-2.0
     7  //
     8  // Unless required by applicable law or agreed to in writing, software
     9  // distributed under the License is distributed on an "AS IS" BASIS,
    10  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package data
    15  
    16  import (
    17  	"bytes"
    18  	"io/ioutil"
    19  	"net/http"
    20  	"net/url"
    21  	"path/filepath"
    22  	"time"
    23  
    24  	"github.com/pkg/errors"
    25  
    26  	"github.com/gohugoio/hugo/cache/filecache"
    27  
    28  	"github.com/gohugoio/hugo/config"
    29  	"github.com/gohugoio/hugo/helpers"
    30  	"github.com/spf13/afero"
    31  )
    32  
    33  var (
    34  	resSleep   = time.Second * 2 // if JSON decoding failed sleep for n seconds before retrying
    35  	resRetries = 1               // number of retries to load the JSON from URL
    36  )
    37  
    38  // getRemote loads the content of a remote file. This method is thread safe.
    39  func (ns *Namespace) getRemote(cache *filecache.Cache, unmarshal func([]byte) (bool, error), req *http.Request) error {
    40  	url := req.URL.String()
    41  	var headers bytes.Buffer
    42  	req.Header.Write(&headers)
    43  	id := helpers.MD5String(url + headers.String())
    44  	var handled bool
    45  	var retry bool
    46  
    47  	_, b, err := cache.GetOrCreateBytes(id, func() ([]byte, error) {
    48  		var err error
    49  		handled = true
    50  		for i := 0; i <= resRetries; i++ {
    51  			ns.deps.Log.Infof("Downloading: %s ...", url)
    52  			var res *http.Response
    53  			res, err = ns.client.Do(req)
    54  			if err != nil {
    55  				return nil, err
    56  			}
    57  
    58  			var b []byte
    59  			b, err = ioutil.ReadAll(res.Body)
    60  			if err != nil {
    61  				return nil, err
    62  			}
    63  			res.Body.Close()
    64  
    65  			if isHTTPError(res) {
    66  				return nil, errors.Errorf("Failed to retrieve remote file: %s, body: %q", http.StatusText(res.StatusCode), b)
    67  			}
    68  
    69  			retry, err = unmarshal(b)
    70  
    71  			if err == nil {
    72  				// Return it so it can be cached.
    73  				return b, nil
    74  			}
    75  
    76  			if !retry {
    77  				return nil, err
    78  			}
    79  
    80  			ns.deps.Log.Infof("Cannot read remote resource %s: %s", url, err)
    81  			ns.deps.Log.Infof("Retry #%d for %s and sleeping for %s", i+1, url, resSleep)
    82  			time.Sleep(resSleep)
    83  		}
    84  
    85  		return nil, err
    86  	})
    87  
    88  	if !handled {
    89  		// This is cached content and should be correct.
    90  		_, err = unmarshal(b)
    91  	}
    92  
    93  	return err
    94  }
    95  
    96  // getLocal loads the content of a local file
    97  func getLocal(url string, fs afero.Fs, cfg config.Provider) ([]byte, error) {
    98  	filename := filepath.Join(cfg.GetString("workingDir"), url)
    99  	return afero.ReadFile(fs, filename)
   100  }
   101  
   102  // getResource loads the content of a local or remote file and returns its content and the
   103  // cache ID used, if relevant.
   104  func (ns *Namespace) getResource(cache *filecache.Cache, unmarshal func(b []byte) (bool, error), req *http.Request) error {
   105  	switch req.URL.Scheme {
   106  	case "":
   107  		url, err := url.QueryUnescape(req.URL.String())
   108  		if err != nil {
   109  			return err
   110  		}
   111  		b, err := getLocal(url, ns.deps.Fs.Source, ns.deps.Cfg)
   112  		if err != nil {
   113  			return err
   114  		}
   115  		_, err = unmarshal(b)
   116  		return err
   117  	default:
   118  		return ns.getRemote(cache, unmarshal, req)
   119  	}
   120  }
   121  
   122  func isHTTPError(res *http.Response) bool {
   123  	return res.StatusCode < 200 || res.StatusCode > 299
   124  }