github.com/graemephi/kahugo@v0.62.3-0.20211121071557-d78c0423784d/tpl/data/data.go (about) 1 // Copyright 2017 The Hugo Authors. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // http://www.apache.org/licenses/LICENSE-2.0 7 // 8 // Unless required by applicable law or agreed to in writing, software 9 // distributed under the License is distributed on an "AS IS" BASIS, 10 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 // See the License for the specific language governing permissions and 12 // limitations under the License. 13 14 // Package data provides template functions for working with external data 15 // sources. 16 package data 17 18 import ( 19 "bytes" 20 "encoding/csv" 21 "encoding/json" 22 "errors" 23 "net/http" 24 "strings" 25 26 "github.com/gohugoio/hugo/common/maps" 27 28 "github.com/gohugoio/hugo/common/types" 29 30 "github.com/gohugoio/hugo/common/constants" 31 "github.com/gohugoio/hugo/common/loggers" 32 33 "github.com/spf13/cast" 34 35 "github.com/gohugoio/hugo/cache/filecache" 36 "github.com/gohugoio/hugo/deps" 37 _errors "github.com/pkg/errors" 38 ) 39 40 // New returns a new instance of the data-namespaced template functions. 41 func New(deps *deps.Deps) *Namespace { 42 return &Namespace{ 43 deps: deps, 44 cacheGetCSV: deps.FileCaches.GetCSVCache(), 45 cacheGetJSON: deps.FileCaches.GetJSONCache(), 46 client: http.DefaultClient, 47 } 48 } 49 50 // Namespace provides template functions for the "data" namespace. 51 type Namespace struct { 52 deps *deps.Deps 53 54 cacheGetJSON *filecache.Cache 55 cacheGetCSV *filecache.Cache 56 57 client *http.Client 58 } 59 60 // GetCSV expects a data separator and one or n-parts of a URL to a resource which 61 // can either be a local or a remote one. 62 // The data separator can be a comma, semi-colon, pipe, etc, but only one character. 63 // If you provide multiple parts for the URL they will be joined together to the final URL. 64 // GetCSV returns nil or a slice slice to use in a short code. 65 func (ns *Namespace) GetCSV(sep string, args ...interface{}) (d [][]string, err error) { 66 url, headers := toURLAndHeaders(args) 67 cache := ns.cacheGetCSV 68 69 unmarshal := func(b []byte) (bool, error) { 70 if d, err = parseCSV(b, sep); err != nil { 71 err = _errors.Wrapf(err, "failed to parse CSV file %s", url) 72 73 return true, err 74 } 75 76 return false, nil 77 } 78 79 var req *http.Request 80 req, err = http.NewRequest("GET", url, nil) 81 if err != nil { 82 return nil, _errors.Wrapf(err, "failed to create request for getCSV for resource %s", url) 83 } 84 85 // Add custom user headers. 86 addUserProvidedHeaders(headers, req) 87 addDefaultHeaders(req, "text/csv", "text/plain") 88 89 err = ns.getResource(cache, unmarshal, req) 90 if err != nil { 91 ns.deps.Log.(loggers.IgnorableLogger).Errorsf(constants.ErrRemoteGetCSV, "Failed to get CSV resource %q: %s", url, err) 92 return nil, nil 93 } 94 95 return 96 } 97 98 // GetJSON expects one or n-parts of a URL to a resource which can either be a local or a remote one. 99 // If you provide multiple parts they will be joined together to the final URL. 100 // GetJSON returns nil or parsed JSON to use in a short code. 101 func (ns *Namespace) GetJSON(args ...interface{}) (interface{}, error) { 102 var v interface{} 103 url, headers := toURLAndHeaders(args) 104 cache := ns.cacheGetJSON 105 106 req, err := http.NewRequest("GET", url, nil) 107 if err != nil { 108 return nil, _errors.Wrapf(err, "Failed to create request for getJSON resource %s", url) 109 } 110 111 unmarshal := func(b []byte) (bool, error) { 112 err := json.Unmarshal(b, &v) 113 if err != nil { 114 return true, err 115 } 116 return false, nil 117 } 118 119 addUserProvidedHeaders(headers, req) 120 addDefaultHeaders(req, "application/json") 121 122 err = ns.getResource(cache, unmarshal, req) 123 if err != nil { 124 ns.deps.Log.(loggers.IgnorableLogger).Errorsf(constants.ErrRemoteGetJSON, "Failed to get JSON resource %q: %s", url, err) 125 return nil, nil 126 } 127 128 return v, nil 129 } 130 131 func addDefaultHeaders(req *http.Request, accepts ...string) { 132 for _, accept := range accepts { 133 if !hasHeaderValue(req.Header, "Accept", accept) { 134 req.Header.Add("Accept", accept) 135 } 136 } 137 if !hasHeaderKey(req.Header, "User-Agent") { 138 req.Header.Add("User-Agent", "Hugo Static Site Generator") 139 } 140 } 141 142 func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) { 143 if headers == nil { 144 return 145 } 146 for key, val := range headers { 147 vals := types.ToStringSlicePreserveString(val) 148 for _, s := range vals { 149 req.Header.Add(key, s) 150 } 151 } 152 } 153 154 func hasHeaderValue(m http.Header, key, value string) bool { 155 var s []string 156 var ok bool 157 158 if s, ok = m[key]; !ok { 159 return false 160 } 161 162 for _, v := range s { 163 if v == value { 164 return true 165 } 166 } 167 return false 168 } 169 170 func hasHeaderKey(m http.Header, key string) bool { 171 _, ok := m[key] 172 return ok 173 } 174 175 func toURLAndHeaders(urlParts []interface{}) (string, map[string]interface{}) { 176 if len(urlParts) == 0 { 177 return "", nil 178 } 179 180 // The last argument may be a map. 181 headers, err := maps.ToStringMapE(urlParts[len(urlParts)-1]) 182 if err == nil { 183 urlParts = urlParts[:len(urlParts)-1] 184 } else { 185 headers = nil 186 } 187 188 return strings.Join(cast.ToStringSlice(urlParts), ""), headers 189 } 190 191 // parseCSV parses bytes of CSV data into a slice slice string or an error 192 func parseCSV(c []byte, sep string) ([][]string, error) { 193 if len(sep) != 1 { 194 return nil, errors.New("Incorrect length of CSV separator: " + sep) 195 } 196 b := bytes.NewReader(c) 197 r := csv.NewReader(b) 198 rSep := []rune(sep) 199 r.Comma = rSep[0] 200 r.FieldsPerRecord = 0 201 return r.ReadAll() 202 }