github.com/paketo-buildpacks/libpak/v2@v2.0.0-alpha.3.0.20231023030503-8365f81de65a/dependency_cache.go (about) 1 /* 2 * Copyright 2018-2023 the original author or authors. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * https://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package libpak 18 19 import ( 20 "crypto/sha256" 21 "encoding/hex" 22 "fmt" 23 "io" 24 "net" 25 "net/http" 26 "net/url" 27 "os" 28 "path/filepath" 29 "strconv" 30 "strings" 31 "time" 32 33 "github.com/BurntSushi/toml" 34 "github.com/buildpacks/libcnb/v2" 35 "github.com/heroku/color" 36 37 "github.com/paketo-buildpacks/libpak/v2/log" 38 "github.com/paketo-buildpacks/libpak/v2/sherpa" 39 ) 40 41 type HttpClientTimeouts struct { 42 DialerTimeout time.Duration 43 DialerKeepAlive time.Duration 44 TLSHandshakeTimeout time.Duration 45 ResponseHeaderTimeout time.Duration 46 ExpectContinueTimeout time.Duration 47 } 48 49 // DependencyCache allows a user to get an artifact either from a buildmodule's cache, a previous download, or to download 50 // directly. 51 type DependencyCache struct { 52 // CachePath is the location where the buildmodule has cached its dependencies. 53 CachePath string 54 55 // DownloadPath is the location of all downloads during this execution of the build. 56 DownloadPath string 57 58 // Logger is the logger used to write to the console. 59 Logger log.Logger 60 61 // UserAgent is the User-Agent string to use with requests. 62 UserAgent string 63 64 // Mappings optionally provides URIs mapping for BuildModuleDependencies 65 Mappings map[string]string 66 67 // httpClientTimeouts contains the timeout values used by HTTP client 68 HttpClientTimeouts HttpClientTimeouts 69 } 70 71 // NewDependencyCache creates a new instance setting the default cache path (<BUILDMODULE_PATH>/dependencies) and user 72 // agent (<BUILDMODULE_ID>/<BUILDMODULE_VERSION>). 73 func NewDependencyCache(buildModuleID string, buildModuleVersion string, buildModulePath string, platformBindings libcnb.Bindings, logger log.Logger) (DependencyCache, error) { 74 cache := DependencyCache{ 75 CachePath: filepath.Join(buildModulePath, "dependencies"), 76 DownloadPath: os.TempDir(), 77 Logger: logger, 78 Mappings: map[string]string{}, 79 UserAgent: fmt.Sprintf("%s/%s", buildModuleID, buildModuleVersion), 80 } 81 mappings, err := mappingsFromBindings(platformBindings) 82 if err != nil { 83 return DependencyCache{}, fmt.Errorf("unable to process dependency-mapping bindings\n%w", err) 84 } 85 cache.Mappings = mappings 86 87 clientTimeouts, err := customizeHttpClientTimeouts() 88 if err != nil { 89 return DependencyCache{}, fmt.Errorf("unable to read custom timeout settings\n%w", err) 90 } 91 cache.HttpClientTimeouts = *clientTimeouts 92 93 return cache, nil 94 } 95 96 func customizeHttpClientTimeouts() (*HttpClientTimeouts, error) { 97 rawStr := sherpa.GetEnvWithDefault("BP_DIALER_TIMEOUT", "6") 98 dialerTimeout, err := strconv.Atoi(rawStr) 99 if err != nil { 100 return nil, fmt.Errorf("unable to convert BP_DIALER_TIMEOUT=%s to integer\n%w", rawStr, err) 101 } 102 103 rawStr = sherpa.GetEnvWithDefault("BP_DIALER_KEEP_ALIVE", "60") 104 dialerKeepAlive, err := strconv.Atoi(rawStr) 105 if err != nil { 106 return nil, fmt.Errorf("unable to convert BP_DIALER_KEEP_ALIVE=%s to integer\n%w", rawStr, err) 107 } 108 109 rawStr = sherpa.GetEnvWithDefault("BP_TLS_HANDSHAKE_TIMEOUT", "5") 110 tlsHandshakeTimeout, err := strconv.Atoi(rawStr) 111 if err != nil { 112 return nil, fmt.Errorf("unable to convert BP_TLS_HANDSHAKE_TIMEOUT=%s to integer\n%w", rawStr, err) 113 } 114 115 rawStr = sherpa.GetEnvWithDefault("BP_RESPONSE_HEADER_TIMEOUT", "5") 116 responseHeaderTimeout, err := strconv.Atoi(rawStr) 117 if err != nil { 118 return nil, fmt.Errorf("unable to convert BP_RESPONSE_HEADER_TIMEOUT=%s to integer\n%w", rawStr, err) 119 } 120 121 rawStr = sherpa.GetEnvWithDefault("BP_EXPECT_CONTINUE_TIMEOUT", "1") 122 expectContinueTimeout, err := strconv.Atoi(rawStr) 123 if err != nil { 124 return nil, fmt.Errorf("unable to convert BP_EXPECT_CONTINUE_TIMEOUT=%s to integer\n%w", rawStr, err) 125 } 126 127 return &HttpClientTimeouts{ 128 DialerTimeout: time.Duration(dialerTimeout) * time.Second, 129 DialerKeepAlive: time.Duration(dialerKeepAlive) * time.Second, 130 TLSHandshakeTimeout: time.Duration(tlsHandshakeTimeout) * time.Second, 131 ResponseHeaderTimeout: time.Duration(responseHeaderTimeout) * time.Second, 132 ExpectContinueTimeout: time.Duration(expectContinueTimeout) * time.Second, 133 }, nil 134 } 135 136 func mappingsFromBindings(bindings libcnb.Bindings) (map[string]string, error) { 137 mappings := map[string]string{} 138 for _, binding := range bindings { 139 if strings.ToLower(binding.Type) == "dependency-mapping" { 140 for digest, uri := range binding.Secret { 141 if _, ok := mappings[digest]; ok { 142 return nil, fmt.Errorf("multiple mappings for digest %q", digest) 143 } 144 mappings[digest] = uri 145 } 146 } 147 } 148 return mappings, nil 149 } 150 151 // RequestModifierFunc is a callback that enables modification of a download request before it is sent. It is often 152 // used to set Authorization headers. 153 type RequestModifierFunc func(request *http.Request) (*http.Request, error) 154 155 // Artifact returns the path to the artifact. Resolution of that path follows three tiers: 156 // 157 // 1. CachePath 158 // 2. DownloadPath 159 // 3. Download from URI 160 // 161 // If the BuildpackDependency's SHA256 is not set, the download can never be verified to be up to date and will always 162 // download, skipping all the caches. 163 func (d *DependencyCache) Artifact(dependency BuildModuleDependency, mods ...RequestModifierFunc) (*os.File, error) { 164 var ( 165 artifact string 166 file string 167 uri = dependency.URI 168 ) 169 170 for d, u := range d.Mappings { 171 if d == dependency.SHA256 { 172 uri = u 173 break 174 } 175 } 176 177 if dependency.SHA256 == "" { 178 d.Logger.Headerf("%s Dependency has no SHA256. Skipping cache.", 179 color.New(color.FgYellow, color.Bold).Sprint("Warning:")) 180 181 d.Logger.Bodyf("%s from %s", color.YellowString("Downloading"), uri) 182 artifact = filepath.Join(d.DownloadPath, filepath.Base(uri)) 183 if err := d.download(uri, artifact, mods...); err != nil { 184 return nil, fmt.Errorf("unable to download %s\n%w", uri, err) 185 } 186 187 return os.Open(artifact) 188 } 189 190 file = filepath.Join(d.CachePath, fmt.Sprintf("%s.toml", dependency.SHA256)) 191 exists, err := sherpa.Exists(file) 192 193 if err != nil { 194 return nil, fmt.Errorf("unable to read %s\n%w", file, err) 195 } 196 197 if exists { 198 d.Logger.Bodyf("%s cached download from buildpack", color.GreenString("Reusing")) 199 return os.Open(filepath.Join(d.CachePath, dependency.SHA256, filepath.Base(uri))) 200 } 201 202 file = filepath.Join(d.DownloadPath, fmt.Sprintf("%s.toml", dependency.SHA256)) 203 exists, err = sherpa.Exists(file) 204 205 if err != nil { 206 return nil, fmt.Errorf("unable to read %s\n%w", file, err) 207 } 208 209 if exists { 210 d.Logger.Bodyf("%s previously cached download", color.GreenString("Reusing")) 211 return os.Open(filepath.Join(d.DownloadPath, dependency.SHA256, filepath.Base(uri))) 212 } 213 214 d.Logger.Bodyf("%s from %s", color.YellowString("Downloading"), uri) 215 artifact = filepath.Join(d.DownloadPath, dependency.SHA256, filepath.Base(uri)) 216 if err := d.download(uri, artifact, mods...); err != nil { 217 return nil, fmt.Errorf("unable to download %s\n%w", uri, err) 218 } 219 220 d.Logger.Body("Verifying checksum") 221 if err := d.verify(artifact, dependency.SHA256); err != nil { 222 return nil, err 223 } 224 225 file = filepath.Join(d.DownloadPath, fmt.Sprintf("%s.toml", dependency.SHA256)) 226 if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil { 227 return nil, fmt.Errorf("unable to make directory %s\n%w", filepath.Dir(file), err) 228 } 229 230 out, err := os.OpenFile(file, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0755) 231 if err != nil { 232 return nil, fmt.Errorf("unable to open file %s\n%w", file, err) 233 } 234 defer out.Close() 235 236 if err := toml.NewEncoder(out).Encode(dependency); err != nil { 237 return nil, fmt.Errorf("unable to write metadata %s\n%w", file, err) 238 } 239 240 return os.Open(artifact) 241 } 242 243 func (d DependencyCache) download(uri string, destination string, mods ...RequestModifierFunc) error { 244 url, err := url.Parse(uri) 245 if err != nil { 246 return fmt.Errorf("unable to parse URI %s\n%w", uri, err) 247 } 248 249 if url.Scheme == "file" { 250 return d.downloadFile(url.Path, destination, mods...) 251 } 252 253 return d.downloadHttp(uri, destination, mods...) 254 } 255 256 func (d DependencyCache) downloadFile(source string, destination string, mods ...RequestModifierFunc) error { 257 if err := os.MkdirAll(filepath.Dir(destination), 0755); err != nil { 258 return fmt.Errorf("unable to make directory %s\n%w", filepath.Dir(destination), err) 259 } 260 261 out, err := os.OpenFile(destination, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 262 if err != nil { 263 return fmt.Errorf("unable to open destination file %s\n%w", destination, err) 264 } 265 defer out.Close() 266 267 input, err := os.Open(source) 268 if err != nil { 269 return fmt.Errorf("unable to open source file %s\n%w", source, err) 270 } 271 defer out.Close() 272 273 if _, err := io.Copy(out, input); err != nil { 274 return fmt.Errorf("unable to copy from %s to %s\n%w", source, destination, err) 275 } 276 277 return nil 278 } 279 280 func (d DependencyCache) downloadHttp(uri string, destination string, mods ...RequestModifierFunc) error { 281 req, err := http.NewRequest("GET", uri, nil) 282 if err != nil { 283 return fmt.Errorf("unable to create new GET request for %s\n%w", uri, err) 284 } 285 286 if d.UserAgent != "" { 287 req.Header.Set("User-Agent", d.UserAgent) 288 } 289 290 for _, m := range mods { 291 req, err = m(req) 292 if err != nil { 293 return fmt.Errorf("unable to modify request\n%w", err) 294 } 295 } 296 297 client := http.Client{ 298 Transport: &http.Transport{ 299 Dial: (&net.Dialer{ 300 Timeout: d.HttpClientTimeouts.DialerTimeout, 301 KeepAlive: d.HttpClientTimeouts.DialerKeepAlive, 302 }).Dial, 303 TLSHandshakeTimeout: d.HttpClientTimeouts.TLSHandshakeTimeout, 304 ResponseHeaderTimeout: d.HttpClientTimeouts.ResponseHeaderTimeout, 305 ExpectContinueTimeout: d.HttpClientTimeouts.ExpectContinueTimeout, 306 Proxy: http.ProxyFromEnvironment, 307 }, 308 } 309 resp, err := client.Do(req) 310 if err != nil { 311 return fmt.Errorf("unable to request %s\n%w", uri, err) 312 } 313 defer resp.Body.Close() 314 315 if resp.StatusCode < 200 || resp.StatusCode > 299 { 316 return fmt.Errorf("could not download %s: %d", uri, resp.StatusCode) 317 } 318 319 if err := os.MkdirAll(filepath.Dir(destination), 0755); err != nil { 320 return fmt.Errorf("unable to make directory %s\n%w", filepath.Dir(destination), err) 321 } 322 323 out, err := os.OpenFile(destination, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 324 if err != nil { 325 return fmt.Errorf("unable to open file %s\n%w", destination, err) 326 } 327 defer out.Close() 328 329 if _, err := io.Copy(out, resp.Body); err != nil { 330 return fmt.Errorf("unable to copy from %s to %s\n%w", uri, destination, err) 331 } 332 333 return nil 334 } 335 336 func (DependencyCache) verify(path string, expected string) error { 337 s := sha256.New() 338 339 in, err := os.Open(path) 340 if err != nil { 341 return fmt.Errorf("unable to verify %s\n%w", path, err) 342 } 343 defer in.Close() 344 345 if _, err := io.Copy(s, in); err != nil { 346 return fmt.Errorf("unable to read %s\n%w", path, err) 347 } 348 349 actual := hex.EncodeToString(s.Sum(nil)) 350 351 if expected != actual { 352 return fmt.Errorf("sha256 for %s %s does not match expected %s", path, actual, expected) 353 } 354 355 return nil 356 }