github.com/remind101/go-getter@v0.0.0-20180809191950-4bda8fa99001/get_http.go (about) 1 package getter 2 3 import ( 4 "encoding/xml" 5 "fmt" 6 "io" 7 "net/http" 8 "net/url" 9 "os" 10 "path/filepath" 11 "strings" 12 13 "github.com/hashicorp/go-safetemp" 14 ) 15 16 // HttpGetter is a Getter implementation that will download from an HTTP 17 // endpoint. 18 // 19 // For file downloads, HTTP is used directly. 20 // 21 // The protocol for downloading a directory from an HTTP endpoing is as follows: 22 // 23 // An HTTP GET request is made to the URL with the additional GET parameter 24 // "terraform-get=1". This lets you handle that scenario specially if you 25 // wish. The response must be a 2xx. 26 // 27 // First, a header is looked for "X-Terraform-Get" which should contain 28 // a source URL to download. 29 // 30 // If the header is not present, then a meta tag is searched for named 31 // "terraform-get" and the content should be a source URL. 32 // 33 // The source URL, whether from the header or meta tag, must be a fully 34 // formed URL. The shorthand syntax of "github.com/foo/bar" or relative 35 // paths are not allowed. 36 type HttpGetter struct { 37 // Netrc, if true, will lookup and use auth information found 38 // in the user's netrc file if available. 39 Netrc bool 40 41 // Client is the http.Client to use for Get requests. 42 // This defaults to a cleanhttp.DefaultClient if left unset. 43 Client *http.Client 44 } 45 46 func (g *HttpGetter) ClientMode(u *url.URL) (ClientMode, error) { 47 if strings.HasSuffix(u.Path, "/") { 48 return ClientModeDir, nil 49 } 50 return ClientModeFile, nil 51 } 52 53 func (g *HttpGetter) Get(dst string, u *url.URL) error { 54 // Copy the URL so we can modify it 55 var newU url.URL = *u 56 u = &newU 57 58 if g.Netrc { 59 // Add auth from netrc if we can 60 if err := addAuthFromNetrc(u); err != nil { 61 return err 62 } 63 } 64 65 if g.Client == nil { 66 g.Client = httpClient 67 } 68 69 // Add terraform-get to the parameter. 70 q := u.Query() 71 q.Add("terraform-get", "1") 72 u.RawQuery = q.Encode() 73 74 // Get the URL 75 resp, err := g.Client.Get(u.String()) 76 if err != nil { 77 return err 78 } 79 defer resp.Body.Close() 80 if resp.StatusCode < 200 || resp.StatusCode >= 300 { 81 return fmt.Errorf("bad response code: %d", resp.StatusCode) 82 } 83 84 // Extract the source URL 85 var source string 86 if v := resp.Header.Get("X-Terraform-Get"); v != "" { 87 source = v 88 } else { 89 source, err = g.parseMeta(resp.Body) 90 if err != nil { 91 return err 92 } 93 } 94 if source == "" { 95 return fmt.Errorf("no source URL was returned") 96 } 97 98 // If there is a subdir component, then we download the root separately 99 // into a temporary directory, then copy over the proper subdir. 100 source, subDir := SourceDirSubdir(source) 101 if subDir == "" { 102 return Get(dst, source) 103 } 104 105 // We have a subdir, time to jump some hoops 106 return g.getSubdir(dst, source, subDir) 107 } 108 109 func (g *HttpGetter) GetFile(dst string, u *url.URL) error { 110 if g.Netrc { 111 // Add auth from netrc if we can 112 if err := addAuthFromNetrc(u); err != nil { 113 return err 114 } 115 } 116 117 if g.Client == nil { 118 g.Client = httpClient 119 } 120 121 resp, err := g.Client.Get(u.String()) 122 if err != nil { 123 return err 124 } 125 defer resp.Body.Close() 126 if resp.StatusCode != 200 { 127 return fmt.Errorf("bad response code: %d", resp.StatusCode) 128 } 129 130 // Create all the parent directories 131 if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil { 132 return err 133 } 134 135 f, err := os.Create(dst) 136 if err != nil { 137 return err 138 } 139 140 n, err := io.Copy(f, resp.Body) 141 if err == nil && n < resp.ContentLength { 142 err = io.ErrShortWrite 143 } 144 if err1 := f.Close(); err == nil { 145 err = err1 146 } 147 return err 148 } 149 150 // getSubdir downloads the source into the destination, but with 151 // the proper subdir. 152 func (g *HttpGetter) getSubdir(dst, source, subDir string) error { 153 // Create a temporary directory to store the full source. This has to be 154 // a non-existent directory. 155 td, tdcloser, err := safetemp.Dir("", "getter") 156 if err != nil { 157 return err 158 } 159 defer tdcloser.Close() 160 161 // Download that into the given directory 162 if err := Get(td, source); err != nil { 163 return err 164 } 165 166 // Process any globbing 167 sourcePath, err := SubdirGlob(td, subDir) 168 if err != nil { 169 return err 170 } 171 172 // Make sure the subdir path actually exists 173 if _, err := os.Stat(sourcePath); err != nil { 174 return fmt.Errorf( 175 "Error downloading %s: %s", source, err) 176 } 177 178 // Copy the subdirectory into our actual destination. 179 if err := os.RemoveAll(dst); err != nil { 180 return err 181 } 182 183 // Make the final destination 184 if err := os.MkdirAll(dst, 0755); err != nil { 185 return err 186 } 187 188 return copyDir(dst, sourcePath, false) 189 } 190 191 // parseMeta looks for the first meta tag in the given reader that 192 // will give us the source URL. 193 func (g *HttpGetter) parseMeta(r io.Reader) (string, error) { 194 d := xml.NewDecoder(r) 195 d.CharsetReader = charsetReader 196 d.Strict = false 197 var err error 198 var t xml.Token 199 for { 200 t, err = d.Token() 201 if err != nil { 202 if err == io.EOF { 203 err = nil 204 } 205 return "", err 206 } 207 if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") { 208 return "", nil 209 } 210 if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") { 211 return "", nil 212 } 213 e, ok := t.(xml.StartElement) 214 if !ok || !strings.EqualFold(e.Name.Local, "meta") { 215 continue 216 } 217 if attrValue(e.Attr, "name") != "terraform-get" { 218 continue 219 } 220 if f := attrValue(e.Attr, "content"); f != "" { 221 return f, nil 222 } 223 } 224 } 225 226 // attrValue returns the attribute value for the case-insensitive key 227 // `name', or the empty string if nothing is found. 228 func attrValue(attrs []xml.Attr, name string) string { 229 for _, a := range attrs { 230 if strings.EqualFold(a.Name.Local, name) { 231 return a.Value 232 } 233 } 234 return "" 235 } 236 237 // charsetReader returns a reader for the given charset. Currently 238 // it only supports UTF-8 and ASCII. Otherwise, it returns a meaningful 239 // error which is printed by go get, so the user can find why the package 240 // wasn't downloaded if the encoding is not supported. Note that, in 241 // order to reduce potential errors, ASCII is treated as UTF-8 (i.e. characters 242 // greater than 0x7f are not rejected). 243 func charsetReader(charset string, input io.Reader) (io.Reader, error) { 244 switch strings.ToLower(charset) { 245 case "ascii": 246 return input, nil 247 default: 248 return nil, fmt.Errorf("can't decode XML document using charset %q", charset) 249 } 250 }