github.com/paketo-buildpacks/libpak@v1.70.0/dependency_cache.go (about) 1 /* 2 * Copyright 2018-2020 the original author or authors. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * https://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package libpak 18 19 import ( 20 "crypto/sha256" 21 "crypto/tls" 22 "encoding/hex" 23 "fmt" 24 "io" 25 "net" 26 "net/http" 27 "net/url" 28 "os" 29 "path/filepath" 30 "strconv" 31 "strings" 32 "time" 33 34 "github.com/BurntSushi/toml" 35 "github.com/buildpacks/libcnb" 36 "github.com/heroku/color" 37 38 "github.com/paketo-buildpacks/libpak/bard" 39 "github.com/paketo-buildpacks/libpak/sherpa" 40 ) 41 42 type HttpClientTimeouts struct { 43 DialerTimeout time.Duration 44 DialerKeepAlive time.Duration 45 TLSHandshakeTimeout time.Duration 46 ResponseHeaderTimeout time.Duration 47 ExpectContinueTimeout time.Duration 48 } 49 50 // DependencyCache allows a user to get an artifact either from a buildpack's cache, a previous download, 51 // a mirror registry, or to download directly. 52 type DependencyCache struct { 53 54 // CachePath is the location where the buildpack has cached its dependencies. 55 CachePath string 56 57 // DownloadPath is the location of all downloads during this execution of the build. 58 DownloadPath string 59 60 // Logger is the logger used to write to the console. 61 Logger bard.Logger 62 63 // UserAgent is the User-Agent string to use with requests. 64 UserAgent string 65 66 // Mappings optionally provides URIs mapping for BuildpackDependencies 67 Mappings map[string]string 68 69 // httpClientTimeouts contains the timeout values used by HTTP client 70 HttpClientTimeouts HttpClientTimeouts 71 72 // Alternative sources used for downloading dependencies. 73 DependencyMirrors map[string]string 74 } 75 76 // NewDependencyCache creates a new instance setting the default cache path (<BUILDPACK_PATH>/dependencies) and user 77 // agent (<BUILDPACK_ID>/<BUILDPACK_VERSION>). 78 // Mappings will be read from any libcnb.Binding in the context with type "dependency-mappings". 79 // 80 // In some environments, many dependencies might need to be downloaded from a (local) mirror registry or filesystem. 81 // Such alternative locations can be configured using bindings of type "dependency-mirror", avoiding too many "dependency-mapping" bindings. 82 // Environment variables named "BP_DEPENDENCY_MIRROR" (default) or "BP_DEPENDENCY_MIRROR_<HOSTNAME>" (hostname-specific mirror) 83 // can also be used for the same purpose. 84 func NewDependencyCache(context libcnb.BuildContext) (DependencyCache, error) { 85 cache := DependencyCache{ 86 CachePath: filepath.Join(context.Buildpack.Path, "dependencies"), 87 DownloadPath: os.TempDir(), 88 UserAgent: fmt.Sprintf("%s/%s", context.Buildpack.Info.ID, context.Buildpack.Info.Version), 89 Mappings: map[string]string{}, 90 DependencyMirrors: map[string]string{}, 91 // We create the logger here because the initialization process may log some warnings that should be visible to users. 92 // This goes against the usual pattern, which has the user supply the Logger after initialization. 93 // There's no choice though, if we want the warning messages to be visible to users. We should clean this up in v2. 94 Logger: bard.NewLogger(os.Stdout), 95 } 96 mappings, err := filterBindingsByType(context.Platform.Bindings, "dependency-mapping") 97 if err != nil { 98 return DependencyCache{}, fmt.Errorf("unable to process dependency-mapping bindings\n%w", err) 99 } 100 cache.Mappings = mappings 101 102 clientTimeouts, err := customizeHttpClientTimeouts() 103 if err != nil { 104 return DependencyCache{}, fmt.Errorf("unable to read custom timeout settings\n%w", err) 105 } 106 cache.HttpClientTimeouts = *clientTimeouts 107 108 bindingMirrors, err := filterBindingsByType(context.Platform.Bindings, "dependency-mirror") 109 if err != nil { 110 return DependencyCache{}, fmt.Errorf("unable to process dependency-mirror bindings\n%w", err) 111 } 112 cache.setDependencyMirrors(bindingMirrors) 113 114 return cache, nil 115 } 116 117 func customizeHttpClientTimeouts() (*HttpClientTimeouts, error) { 118 rawStr := sherpa.GetEnvWithDefault("BP_DIALER_TIMEOUT", "6") 119 dialerTimeout, err := strconv.Atoi(rawStr) 120 if err != nil { 121 return nil, fmt.Errorf("unable to convert BP_DIALER_TIMEOUT=%s to integer\n%w", rawStr, err) 122 } 123 124 rawStr = sherpa.GetEnvWithDefault("BP_DIALER_KEEP_ALIVE", "60") 125 dialerKeepAlive, err := strconv.Atoi(rawStr) 126 if err != nil { 127 return nil, fmt.Errorf("unable to convert BP_DIALER_KEEP_ALIVE=%s to integer\n%w", rawStr, err) 128 } 129 130 rawStr = sherpa.GetEnvWithDefault("BP_TLS_HANDSHAKE_TIMEOUT", "5") 131 tlsHandshakeTimeout, err := strconv.Atoi(rawStr) 132 if err != nil { 133 return nil, fmt.Errorf("unable to convert BP_TLS_HANDSHAKE_TIMEOUT=%s to integer\n%w", rawStr, err) 134 } 135 136 rawStr = sherpa.GetEnvWithDefault("BP_RESPONSE_HEADER_TIMEOUT", "5") 137 responseHeaderTimeout, err := strconv.Atoi(rawStr) 138 if err != nil { 139 return nil, fmt.Errorf("unable to convert BP_RESPONSE_HEADER_TIMEOUT=%s to integer\n%w", rawStr, err) 140 } 141 142 rawStr = sherpa.GetEnvWithDefault("BP_EXPECT_CONTINUE_TIMEOUT", "1") 143 expectContinueTimeout, err := strconv.Atoi(rawStr) 144 if err != nil { 145 return nil, fmt.Errorf("unable to convert BP_EXPECT_CONTINUE_TIMEOUT=%s to integer\n%w", rawStr, err) 146 } 147 148 return &HttpClientTimeouts{ 149 DialerTimeout: time.Duration(dialerTimeout) * time.Second, 150 DialerKeepAlive: time.Duration(dialerKeepAlive) * time.Second, 151 TLSHandshakeTimeout: time.Duration(tlsHandshakeTimeout) * time.Second, 152 ResponseHeaderTimeout: time.Duration(responseHeaderTimeout) * time.Second, 153 ExpectContinueTimeout: time.Duration(expectContinueTimeout) * time.Second, 154 }, nil 155 } 156 157 func (d *DependencyCache) setDependencyMirrors(bindingMirrors map[string]string) { 158 // Initialize with mirrors from bindings. 159 d.DependencyMirrors = bindingMirrors 160 // Add mirrors from env variables and override duplicate hostnames set in bindings. 161 envs := os.Environ() 162 for _, env := range envs { 163 envPair := strings.SplitN(env, "=", 2) 164 if len(envPair) != 2 { 165 continue 166 } 167 hostnameSuffix, isMirror := strings.CutPrefix(envPair[0], "BP_DEPENDENCY_MIRROR") 168 if isMirror { 169 hostnameEncoded, _ := strings.CutPrefix(hostnameSuffix, "_") 170 if strings.ToLower(hostnameEncoded) == "default" { 171 d.Logger.Bodyf("%s with illegal hostname 'default'. Please use BP_DEPENDENCY_MIRROR to set a default.", 172 color.YellowString("Ignored dependency mirror")) 173 continue 174 } 175 d.DependencyMirrors[decodeHostnameEnv(hostnameEncoded, d)] = envPair[1] 176 } 177 } 178 } 179 180 // Takes an encoded hostname (from env key) and returns the decoded version in lower case. 181 // Replaces double underscores (__) with one dash (-) and single underscores (_) with one period (.). 182 func decodeHostnameEnv(encodedHostname string, d *DependencyCache) string { 183 if strings.ContainsAny(encodedHostname, "-.") || encodedHostname != strings.ToUpper(encodedHostname) { 184 d.Logger.Bodyf("%s These will be allowed but for best results across different shells, you should replace . characters with _ characters "+ 185 "and - characters with __, and use all upper case letters. The buildpack will convert these back before using the mirror.", 186 color.YellowString("You have invalid characters in your mirror host environment variable.")) 187 } 188 var decodedHostname string 189 if encodedHostname == "" { 190 decodedHostname = "default" 191 } else { 192 decodedHostname = strings.ReplaceAll(strings.ReplaceAll(encodedHostname, "__", "-"), "_", ".") 193 } 194 return strings.ToLower(decodedHostname) 195 } 196 197 // Returns a key/value map with all entries for a given binding type. 198 // An error is returned if multiple entries are found using the same key (e.g. duplicate digests in dependency mappings). 199 func filterBindingsByType(bindings libcnb.Bindings, bindingType string) (map[string]string, error) { 200 filteredBindings := map[string]string{} 201 for _, binding := range bindings { 202 if strings.ToLower(binding.Type) == bindingType { 203 for key, value := range binding.Secret { 204 if _, ok := filteredBindings[strings.ToLower(key)]; ok { 205 return nil, fmt.Errorf("multiple %s bindings found with duplicate keys %s", binding.Type, key) 206 } 207 filteredBindings[strings.ToLower(key)] = value 208 } 209 } 210 } 211 return filteredBindings, nil 212 } 213 214 // RequestModifierFunc is a callback that enables modification of a download request before it is sent. It is often 215 // used to set Authorization headers. 216 type RequestModifierFunc func(request *http.Request) (*http.Request, error) 217 218 // Artifact returns the path to the artifact. Resolution of that path follows three tiers: 219 // 220 // 1. CachePath 221 // 2. DownloadPath 222 // 3. Download from URI 223 // 224 // If the BuildpackDependency's SHA256 is not set, the download can never be verified to be up to date and will always 225 // download, skipping all the caches. 226 func (d *DependencyCache) Artifact(dependency BuildpackDependency, mods ...RequestModifierFunc) (*os.File, error) { 227 228 var ( 229 actual BuildpackDependency 230 artifact string 231 file string 232 isBinding bool 233 uri = dependency.URI 234 urlP *url.URL 235 ) 236 237 for d, u := range d.Mappings { 238 if d == dependency.SHA256 { 239 isBinding = true 240 uri = u 241 break 242 } 243 } 244 245 urlP, err := url.Parse(uri) 246 if err != nil { 247 d.Logger.Debugf("URI format invalid\n%w", err) 248 return nil, fmt.Errorf("unable to parse URI. see DEBUG log level") 249 } 250 251 mirror := d.DependencyMirrors["default"] 252 mirrorHostSpecific := d.DependencyMirrors[urlP.Hostname()] 253 if mirrorHostSpecific != "" { 254 mirror = mirrorHostSpecific 255 } 256 257 if isBinding && mirror != "" { 258 d.Logger.Bodyf("Both dependency mirror and bindings are present. %s Please remove dependency map bindings if you wish to use the mirror.", 259 color.YellowString("Mirror is being ignored.")) 260 } else { 261 d.setDependencyMirror(urlP, mirror) 262 } 263 264 if dependency.SHA256 == "" { 265 d.Logger.Headerf("%s Dependency has no SHA256. Skipping cache.", 266 color.New(color.FgYellow, color.Bold).Sprint("Warning:")) 267 268 d.Logger.Bodyf("%s from %s", color.YellowString("Downloading"), urlP.Redacted()) 269 artifact = filepath.Join(d.DownloadPath, filepath.Base(uri)) 270 if err := d.download(urlP, artifact, mods...); err != nil { 271 return nil, fmt.Errorf("unable to download %s\n%w", urlP.Redacted(), err) 272 } 273 274 return os.Open(artifact) 275 } 276 277 file = filepath.Join(d.CachePath, fmt.Sprintf("%s.toml", dependency.SHA256)) 278 b, err := os.ReadFile(file) 279 if err != nil && !os.IsNotExist(err) { 280 return nil, fmt.Errorf("unable to read %s\n%w", file, err) 281 } 282 if err := toml.Unmarshal(b, &actual); err != nil { 283 return nil, fmt.Errorf("unable to decode download metadata %s\n%w", file, err) 284 } 285 286 if dependency.Equals(actual) { 287 d.Logger.Bodyf("%s cached download from buildpack", color.GreenString("Reusing")) 288 return os.Open(filepath.Join(d.CachePath, dependency.SHA256, filepath.Base(urlP.Path))) 289 } 290 291 file = filepath.Join(d.DownloadPath, fmt.Sprintf("%s.toml", dependency.SHA256)) 292 b, err = os.ReadFile(file) 293 if err != nil && !os.IsNotExist(err) { 294 return nil, fmt.Errorf("unable to read %s\n%w", file, err) 295 } 296 if err := toml.Unmarshal(b, &actual); err != nil { 297 return nil, fmt.Errorf("unable to decode download metadata %s\n%w", file, err) 298 } 299 300 if dependency.Equals(actual) { 301 d.Logger.Bodyf("%s previously cached download", color.GreenString("Reusing")) 302 return os.Open(filepath.Join(d.DownloadPath, dependency.SHA256, filepath.Base(urlP.Path))) 303 } 304 305 d.Logger.Bodyf("%s from %s", color.YellowString("Downloading"), urlP.Redacted()) 306 artifact = filepath.Join(d.DownloadPath, dependency.SHA256, filepath.Base(uri)) 307 if err := d.download(urlP, artifact, mods...); err != nil { 308 return nil, fmt.Errorf("unable to download %s\n%w", urlP.Redacted(), err) 309 } 310 311 d.Logger.Body("Verifying checksum") 312 if err := d.verify(artifact, dependency.SHA256); err != nil { 313 return nil, err 314 } 315 316 file = filepath.Join(d.DownloadPath, fmt.Sprintf("%s.toml", dependency.SHA256)) 317 if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil { 318 return nil, fmt.Errorf("unable to make directory %s\n%w", filepath.Dir(file), err) 319 } 320 321 out, err := os.OpenFile(file, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0755) 322 if err != nil { 323 return nil, fmt.Errorf("unable to open file %s\n%w", file, err) 324 } 325 defer out.Close() 326 327 if err := toml.NewEncoder(out).Encode(dependency); err != nil { 328 return nil, fmt.Errorf("unable to write metadata %s\n%w", file, err) 329 } 330 331 return os.Open(artifact) 332 } 333 334 func (d DependencyCache) download(url *url.URL, destination string, mods ...RequestModifierFunc) error { 335 if url.Scheme == "file" { 336 return d.downloadFile(url.Path, destination, mods...) 337 } 338 339 return d.downloadHttp(url, destination, mods...) 340 } 341 342 func (d DependencyCache) downloadFile(source string, destination string, mods ...RequestModifierFunc) error { 343 if err := os.MkdirAll(filepath.Dir(destination), 0755); err != nil { 344 return fmt.Errorf("unable to make directory %s\n%w", filepath.Dir(destination), err) 345 } 346 347 out, err := os.OpenFile(destination, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 348 if err != nil { 349 return fmt.Errorf("unable to open destination file %s\n%w", destination, err) 350 } 351 defer out.Close() 352 353 input, err := os.Open(source) 354 if err != nil { 355 return fmt.Errorf("unable to open source file %s\n%w", source, err) 356 } 357 defer out.Close() 358 359 if _, err := io.Copy(out, input); err != nil { 360 return fmt.Errorf("unable to copy from %s to %s\n%w", source, destination, err) 361 } 362 363 return nil 364 } 365 366 func (d DependencyCache) downloadHttp(url *url.URL, destination string, mods ...RequestModifierFunc) error { 367 var httpClient *http.Client 368 if (strings.EqualFold(url.Hostname(), "localhost")) || (strings.EqualFold(url.Hostname(), "127.0.0.1")) { 369 httpClient = &http.Client{ 370 Transport: &http.Transport{ 371 TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 372 }, 373 } 374 } else { 375 httpClient = &http.Client{ 376 Transport: &http.Transport{ 377 Dial: (&net.Dialer{ 378 Timeout: d.HttpClientTimeouts.DialerTimeout, 379 KeepAlive: d.HttpClientTimeouts.DialerKeepAlive, 380 }).Dial, 381 TLSHandshakeTimeout: d.HttpClientTimeouts.TLSHandshakeTimeout, 382 ResponseHeaderTimeout: d.HttpClientTimeouts.ResponseHeaderTimeout, 383 ExpectContinueTimeout: d.HttpClientTimeouts.ExpectContinueTimeout, 384 Proxy: http.ProxyFromEnvironment, 385 }, 386 } 387 } 388 389 req, err := http.NewRequest("GET", url.String(), nil) 390 if err != nil { 391 return fmt.Errorf("unable to create new GET request for %s\n%w", url.Redacted(), err) 392 } 393 394 if d.UserAgent != "" { 395 req.Header.Set("User-Agent", d.UserAgent) 396 } 397 398 for _, m := range mods { 399 req, err = m(req) 400 if err != nil { 401 return fmt.Errorf("unable to modify request\n%w", err) 402 } 403 } 404 405 resp, err := httpClient.Do(req) 406 if err != nil { 407 return fmt.Errorf("unable to request %s\n%w", url.Redacted(), err) 408 } 409 defer resp.Body.Close() 410 411 if resp.StatusCode < 200 || resp.StatusCode > 299 { 412 return fmt.Errorf("could not download %s: %d", url.Redacted(), resp.StatusCode) 413 } 414 415 if err := os.MkdirAll(filepath.Dir(destination), 0755); err != nil { 416 return fmt.Errorf("unable to make directory %s\n%w", filepath.Dir(destination), err) 417 } 418 419 out, err := os.OpenFile(destination, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 420 if err != nil { 421 return fmt.Errorf("unable to open file %s\n%w", destination, err) 422 } 423 defer out.Close() 424 425 if _, err := io.Copy(out, resp.Body); err != nil { 426 return fmt.Errorf("unable to copy from %s to %s\n%w", url.Redacted(), destination, err) 427 } 428 429 return nil 430 } 431 432 func (DependencyCache) verify(path string, expected string) error { 433 s := sha256.New() 434 435 in, err := os.Open(path) 436 if err != nil { 437 return fmt.Errorf("unable to verify %s\n%w", path, err) 438 } 439 defer in.Close() 440 441 if _, err := io.Copy(s, in); err != nil { 442 return fmt.Errorf("unable to read %s\n%w", path, err) 443 } 444 445 actual := hex.EncodeToString(s.Sum(nil)) 446 447 if expected != actual { 448 return fmt.Errorf("sha256 for %s %s does not match expected %s", path, actual, expected) 449 } 450 451 return nil 452 } 453 454 func (d DependencyCache) setDependencyMirror(urlD *url.URL, mirror string) { 455 if mirror != "" { 456 d.Logger.Bodyf("%s Download URIs will be overridden.", color.GreenString("Dependency mirror found.")) 457 urlOverride, err := url.ParseRequestURI(mirror) 458 459 if strings.ToLower(urlOverride.Scheme) == "https" || strings.ToLower(urlOverride.Scheme) == "file" { 460 urlD.Scheme = urlOverride.Scheme 461 urlD.User = urlOverride.User 462 urlD.Path = strings.Replace(urlOverride.Path, "{originalHost}", urlD.Hostname(), 1) + urlD.Path 463 urlD.Host = urlOverride.Host 464 } else { 465 d.Logger.Debugf("Dependency mirror URI is invalid: %s\n%w", mirror, err) 466 d.Logger.Bodyf("%s is ignored. Have you used one of the supported schemes https:// or file://?", color.YellowString("Invalid dependency mirror")) 467 } 468 } 469 }