github.com/shohhei1126/hugo@v0.42.2-0.20180623210752-3d5928889ad7/tpl/data/resources.go (about) 1 // Copyright 2016 The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 package data 15 16 import ( 17 "fmt" 18 "io/ioutil" 19 "net/http" 20 "path/filepath" 21 "sync" 22 "time" 23 24 "github.com/gohugoio/hugo/config" 25 "github.com/gohugoio/hugo/helpers" 26 "github.com/spf13/afero" 27 jww "github.com/spf13/jwalterweatherman" 28 ) 29 30 var ( 31 remoteURLLock = &remoteLock{m: make(map[string]*sync.Mutex)} 32 resSleep = time.Second * 2 // if JSON decoding failed sleep for n seconds before retrying 33 resRetries = 1 // number of retries to load the JSON from URL or local file system 34 ) 35 36 type remoteLock struct { 37 sync.RWMutex 38 m map[string]*sync.Mutex 39 } 40 41 // URLLock locks an URL during download 42 func (l *remoteLock) URLLock(url string) { 43 var ( 44 lock *sync.Mutex 45 ok bool 46 ) 47 l.Lock() 48 if lock, ok = l.m[url]; !ok { 49 lock = &sync.Mutex{} 50 l.m[url] = lock 51 } 52 l.Unlock() 53 lock.Lock() 54 } 55 56 // URLUnlock unlocks an URL when the download has been finished. Use only in defer calls. 57 func (l *remoteLock) URLUnlock(url string) { 58 l.RLock() 59 defer l.RUnlock() 60 if um, ok := l.m[url]; ok { 61 um.Unlock() 62 } 63 } 64 65 // getRemote loads the content of a remote file. This method is thread safe. 66 func getRemote(req *http.Request, fs afero.Fs, cfg config.Provider, hc *http.Client) ([]byte, error) { 67 url := req.URL.String() 68 69 c, err := getCache(url, fs, cfg, cfg.GetBool("ignoreCache")) 70 if err != nil { 71 return nil, err 72 } 73 if c != nil { 74 return c, nil 75 } 76 77 // avoid race condition with locks, block other goroutines if the current url is processing 78 remoteURLLock.URLLock(url) 79 defer func() { remoteURLLock.URLUnlock(url) }() 80 81 // avoid multiple locks due to calling getCache twice 82 c, err = getCache(url, fs, cfg, cfg.GetBool("ignoreCache")) 83 if err != nil { 84 return nil, err 85 } 86 if c != nil { 87 return c, nil 88 } 89 90 jww.INFO.Printf("Downloading: %s ...", url) 91 res, err := hc.Do(req) 92 if err != nil { 93 return nil, err 94 } 95 96 if res.StatusCode < 200 || res.StatusCode > 299 { 97 return nil, fmt.Errorf("Failed to retrieve remote file: %s", http.StatusText(res.StatusCode)) 98 } 99 100 c, err = ioutil.ReadAll(res.Body) 101 res.Body.Close() 102 if err != nil { 103 return nil, err 104 } 105 106 err = writeCache(url, c, fs, cfg, cfg.GetBool("ignoreCache")) 107 if err != nil { 108 return nil, err 109 } 110 111 jww.INFO.Printf("... and cached to: %s", getCacheFileID(cfg, url)) 112 return c, nil 113 } 114 115 // getLocal loads the content of a local file 116 func getLocal(url string, fs afero.Fs, cfg config.Provider) ([]byte, error) { 117 filename := filepath.Join(cfg.GetString("workingDir"), url) 118 if e, err := helpers.Exists(filename, fs); !e { 119 return nil, err 120 } 121 122 return afero.ReadFile(fs, filename) 123 124 } 125 126 // getResource loads the content of a local or remote file 127 func (ns *Namespace) getResource(req *http.Request) ([]byte, error) { 128 switch req.URL.Scheme { 129 case "": 130 return getLocal(req.URL.String(), ns.deps.Fs.Source, ns.deps.Cfg) 131 default: 132 return getRemote(req, ns.deps.Fs.Source, ns.deps.Cfg, ns.client) 133 } 134 }