github.com/kovansky/hugo@v0.92.3-0.20220224232819-63076e4ff19f/tpl/data/resources.go (about)

     1  // Copyright 2016 The Hugo Authors. All rights reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  // http://www.apache.org/licenses/LICENSE-2.0
     7  //
     8  // Unless required by applicable law or agreed to in writing, software
     9  // distributed under the License is distributed on an "AS IS" BASIS,
    10  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package data
    15  
    16  import (
    17  	"bytes"
    18  	"io/ioutil"
    19  	"net/http"
    20  	"net/url"
    21  	"path/filepath"
    22  	"time"
    23  
    24  	"github.com/pkg/errors"
    25  
    26  	"github.com/gohugoio/hugo/cache/filecache"
    27  
    28  	"github.com/gohugoio/hugo/config"
    29  	"github.com/gohugoio/hugo/helpers"
    30  	"github.com/spf13/afero"
    31  )
    32  
    33  var (
    34  	resSleep   = time.Second * 2 // if JSON decoding failed sleep for n seconds before retrying
    35  	resRetries = 1               // number of retries to load the JSON from URL
    36  )
    37  
    38  // getRemote loads the content of a remote file. This method is thread safe.
    39  func (ns *Namespace) getRemote(cache *filecache.Cache, unmarshal func([]byte) (bool, error), req *http.Request) error {
    40  	url := req.URL.String()
    41  	if err := ns.deps.ExecHelper.Sec().CheckAllowedHTTPURL(url); err != nil {
    42  		return err
    43  	}
    44  	if err := ns.deps.ExecHelper.Sec().CheckAllowedHTTPMethod("GET"); err != nil {
    45  		return err
    46  	}
    47  	
    48  	var headers bytes.Buffer
    49  	req.Header.Write(&headers)
    50  	id := helpers.MD5String(url + headers.String())
    51  	var handled bool
    52  	var retry bool
    53  
    54  	_, b, err := cache.GetOrCreateBytes(id, func() ([]byte, error) {
    55  		var err error
    56  		handled = true
    57  		for i := 0; i <= resRetries; i++ {
    58  			ns.deps.Log.Infof("Downloading: %s ...", url)
    59  			var res *http.Response
    60  			res, err = ns.client.Do(req)
    61  			if err != nil {
    62  				return nil, err
    63  			}
    64  
    65  			var b []byte
    66  			b, err = ioutil.ReadAll(res.Body)
    67  			if err != nil {
    68  				return nil, err
    69  			}
    70  			res.Body.Close()
    71  
    72  			if isHTTPError(res) {
    73  				return nil, errors.Errorf("Failed to retrieve remote file: %s, body: %q", http.StatusText(res.StatusCode), b)
    74  			}
    75  
    76  			retry, err = unmarshal(b)
    77  
    78  			if err == nil {
    79  				// Return it so it can be cached.
    80  				return b, nil
    81  			}
    82  
    83  			if !retry {
    84  				return nil, err
    85  			}
    86  
    87  			ns.deps.Log.Infof("Cannot read remote resource %s: %s", url, err)
    88  			ns.deps.Log.Infof("Retry #%d for %s and sleeping for %s", i+1, url, resSleep)
    89  			time.Sleep(resSleep)
    90  		}
    91  
    92  		return nil, err
    93  	})
    94  
    95  	if !handled {
    96  		// This is cached content and should be correct.
    97  		_, err = unmarshal(b)
    98  	}
    99  
   100  	return err
   101  }
   102  
   103  // getLocal loads the content of a local file
   104  func getLocal(url string, fs afero.Fs, cfg config.Provider) ([]byte, error) {
   105  	filename := filepath.Join(cfg.GetString("workingDir"), url)
   106  	return afero.ReadFile(fs, filename)
   107  }
   108  
   109  // getResource loads the content of a local or remote file and returns its content and the
   110  // cache ID used, if relevant.
   111  func (ns *Namespace) getResource(cache *filecache.Cache, unmarshal func(b []byte) (bool, error), req *http.Request) error {
   112  	switch req.URL.Scheme {
   113  	case "":
   114  		url, err := url.QueryUnescape(req.URL.String())
   115  		if err != nil {
   116  			return err
   117  		}
   118  		b, err := getLocal(url, ns.deps.Fs.Source, ns.deps.Cfg)
   119  		if err != nil {
   120  			return err
   121  		}
   122  		_, err = unmarshal(b)
   123  		return err
   124  	default:
   125  		return ns.getRemote(cache, unmarshal, req)
   126  	}
   127  }
   128  
   129  func isHTTPError(res *http.Response) bool {
   130  	return res.StatusCode < 200 || res.StatusCode > 299
   131  }