github.com/remind101/go-getter@v0.0.0-20180809191950-4bda8fa99001/client.go (about) 1 package getter 2 3 import ( 4 "bytes" 5 "crypto/md5" 6 "crypto/sha1" 7 "crypto/sha256" 8 "crypto/sha512" 9 "encoding/hex" 10 "fmt" 11 "hash" 12 "io" 13 "io/ioutil" 14 "os" 15 "path/filepath" 16 "strconv" 17 "strings" 18 19 urlhelper "github.com/hashicorp/go-getter/helper/url" 20 "github.com/hashicorp/go-safetemp" 21 ) 22 23 // Client is a client for downloading things. 24 // 25 // Top-level functions such as Get are shortcuts for interacting with a client. 26 // Using a client directly allows more fine-grained control over how downloading 27 // is done, as well as customizing the protocols supported. 28 type Client struct { 29 // Src is the source URL to get. 30 // 31 // Dst is the path to save the downloaded thing as. If Dir is set to 32 // true, then this should be a directory. If the directory doesn't exist, 33 // it will be created for you. 34 // 35 // Pwd is the working directory for detection. If this isn't set, some 36 // detection may fail. Client will not default pwd to the current 37 // working directory for security reasons. 38 Src string 39 Dst string 40 Pwd string 41 42 // Mode is the method of download the client will use. See ClientMode 43 // for documentation. 44 Mode ClientMode 45 46 // Detectors is the list of detectors that are tried on the source. 47 // If this is nil, then the default Detectors will be used. 48 Detectors []Detector 49 50 // Decompressors is the map of decompressors supported by this client. 51 // If this is nil, then the default value is the Decompressors global. 52 Decompressors map[string]Decompressor 53 54 // Getters is the map of protocols supported by this client. If this 55 // is nil, then the default Getters variable will be used. 56 Getters map[string]Getter 57 58 // Dir, if true, tells the Client it is downloading a directory (versus 59 // a single file). This distinction is necessary since filenames and 60 // directory names follow the same format so disambiguating is impossible 61 // without knowing ahead of time. 62 // 63 // WARNING: deprecated. If Mode is set, that will take precedence. 64 Dir bool 65 } 66 67 // Get downloads the configured source to the destination. 68 func (c *Client) Get() error { 69 // Store this locally since there are cases we swap this 70 mode := c.Mode 71 if mode == ClientModeInvalid { 72 if c.Dir { 73 mode = ClientModeDir 74 } else { 75 mode = ClientModeFile 76 } 77 } 78 79 // Default decompressor value 80 decompressors := c.Decompressors 81 if decompressors == nil { 82 decompressors = Decompressors 83 } 84 85 // Detect the URL. This is safe if it is already detected. 86 detectors := c.Detectors 87 if detectors == nil { 88 detectors = Detectors 89 } 90 src, err := Detect(c.Src, c.Pwd, detectors) 91 if err != nil { 92 return err 93 } 94 95 // Determine if we have a forced protocol, i.e. "git::http://..." 96 force, src := getForcedGetter(src) 97 98 // If there is a subdir component, then we download the root separately 99 // and then copy over the proper subdir. 100 var realDst string 101 dst := c.Dst 102 src, subDir := SourceDirSubdir(src) 103 if subDir != "" { 104 td, tdcloser, err := safetemp.Dir("", "getter") 105 if err != nil { 106 return err 107 } 108 defer tdcloser.Close() 109 110 realDst = dst 111 dst = td 112 } 113 114 u, err := urlhelper.Parse(src) 115 if err != nil { 116 return err 117 } 118 if force == "" { 119 force = u.Scheme 120 } 121 122 getters := c.Getters 123 if getters == nil { 124 getters = Getters 125 } 126 127 g, ok := getters[force] 128 if !ok { 129 return fmt.Errorf( 130 "download not supported for scheme '%s'", force) 131 } 132 133 // We have magic query parameters that we use to signal different features 134 q := u.Query() 135 136 // Determine if we have an archive type 137 archiveV := q.Get("archive") 138 if archiveV != "" { 139 // Delete the paramter since it is a magic parameter we don't 140 // want to pass on to the Getter 141 q.Del("archive") 142 u.RawQuery = q.Encode() 143 144 // If we can parse the value as a bool and it is false, then 145 // set the archive to "-" which should never map to a decompressor 146 if b, err := strconv.ParseBool(archiveV); err == nil && !b { 147 archiveV = "-" 148 } 149 } 150 if archiveV == "" { 151 // We don't appear to... but is it part of the filename? 152 matchingLen := 0 153 for k, _ := range decompressors { 154 if strings.HasSuffix(u.Path, "."+k) && len(k) > matchingLen { 155 archiveV = k 156 matchingLen = len(k) 157 } 158 } 159 } 160 161 // If we have a decompressor, then we need to change the destination 162 // to download to a temporary path. We unarchive this into the final, 163 // real path. 164 var decompressDst string 165 var decompressDir bool 166 decompressor := decompressors[archiveV] 167 if decompressor != nil { 168 // Create a temporary directory to store our archive. We delete 169 // this at the end of everything. 170 td, err := ioutil.TempDir("", "getter") 171 if err != nil { 172 return fmt.Errorf( 173 "Error creating temporary directory for archive: %s", err) 174 } 175 defer os.RemoveAll(td) 176 177 // Swap the download directory to be our temporary path and 178 // store the old values. 179 decompressDst = dst 180 decompressDir = mode != ClientModeFile 181 dst = filepath.Join(td, "archive") 182 mode = ClientModeFile 183 } 184 185 // Determine if we have a checksum 186 var checksumHash hash.Hash 187 var checksumValue []byte 188 if v := q.Get("checksum"); v != "" { 189 // Delete the query parameter if we have it. 190 q.Del("checksum") 191 u.RawQuery = q.Encode() 192 193 // Determine the checksum hash type 194 checksumType := "" 195 idx := strings.Index(v, ":") 196 if idx > -1 { 197 checksumType = v[:idx] 198 } 199 switch checksumType { 200 case "md5": 201 checksumHash = md5.New() 202 case "sha1": 203 checksumHash = sha1.New() 204 case "sha256": 205 checksumHash = sha256.New() 206 case "sha512": 207 checksumHash = sha512.New() 208 default: 209 return fmt.Errorf( 210 "unsupported checksum type: %s", checksumType) 211 } 212 213 // Get the remainder of the value and parse it into bytes 214 b, err := hex.DecodeString(v[idx+1:]) 215 if err != nil { 216 return fmt.Errorf("invalid checksum: %s", err) 217 } 218 219 // Set our value 220 checksumValue = b 221 } 222 223 if mode == ClientModeAny { 224 // Ask the getter which client mode to use 225 mode, err = g.ClientMode(u) 226 if err != nil { 227 return err 228 } 229 230 // Destination is the base name of the URL path in "any" mode when 231 // a file source is detected. 232 if mode == ClientModeFile { 233 filename := filepath.Base(u.Path) 234 235 // Determine if we have a custom file name 236 if v := q.Get("filename"); v != "" { 237 // Delete the query parameter if we have it. 238 q.Del("filename") 239 u.RawQuery = q.Encode() 240 241 filename = v 242 } 243 244 dst = filepath.Join(dst, filename) 245 } 246 } 247 248 // If we're not downloading a directory, then just download the file 249 // and return. 250 if mode == ClientModeFile { 251 err := g.GetFile(dst, u) 252 if err != nil { 253 return err 254 } 255 256 if checksumHash != nil { 257 if err := checksum(dst, checksumHash, checksumValue); err != nil { 258 return err 259 } 260 } 261 262 if decompressor != nil { 263 // We have a decompressor, so decompress the current destination 264 // into the final destination with the proper mode. 265 err := decompressor.Decompress(decompressDst, dst, decompressDir) 266 if err != nil { 267 return err 268 } 269 270 // Swap the information back 271 dst = decompressDst 272 if decompressDir { 273 mode = ClientModeAny 274 } else { 275 mode = ClientModeFile 276 } 277 } 278 279 // We check the dir value again because it can be switched back 280 // if we were unarchiving. If we're still only Get-ing a file, then 281 // we're done. 282 if mode == ClientModeFile { 283 return nil 284 } 285 } 286 287 // If we're at this point we're either downloading a directory or we've 288 // downloaded and unarchived a directory and we're just checking subdir. 289 // In the case we have a decompressor we don't Get because it was Get 290 // above. 291 if decompressor == nil { 292 // If we're getting a directory, then this is an error. You cannot 293 // checksum a directory. TODO: test 294 if checksumHash != nil { 295 return fmt.Errorf( 296 "checksum cannot be specified for directory download") 297 } 298 299 // We're downloading a directory, which might require a bit more work 300 // if we're specifying a subdir. 301 err := g.Get(dst, u) 302 if err != nil { 303 err = fmt.Errorf("error downloading '%s': %s", src, err) 304 return err 305 } 306 } 307 308 // If we have a subdir, copy that over 309 if subDir != "" { 310 if err := os.RemoveAll(realDst); err != nil { 311 return err 312 } 313 if err := os.MkdirAll(realDst, 0755); err != nil { 314 return err 315 } 316 317 // Process any globs 318 subDir, err := SubdirGlob(dst, subDir) 319 if err != nil { 320 return err 321 } 322 323 return copyDir(realDst, subDir, false) 324 } 325 326 return nil 327 } 328 329 // checksum is a simple method to compute the checksum of a source file 330 // and compare it to the given expected value. 331 func checksum(source string, h hash.Hash, v []byte) error { 332 f, err := os.Open(source) 333 if err != nil { 334 return fmt.Errorf("Failed to open file for checksum: %s", err) 335 } 336 defer f.Close() 337 338 if _, err := io.Copy(h, f); err != nil { 339 return fmt.Errorf("Failed to hash: %s", err) 340 } 341 342 if actual := h.Sum(nil); !bytes.Equal(actual, v) { 343 return fmt.Errorf( 344 "Checksums did not match.\nExpected: %s\nGot: %s", 345 hex.EncodeToString(v), 346 hex.EncodeToString(actual)) 347 } 348 349 return nil 350 }