github.com/BarDweller/libpak@v0.0.0-20230630201634-8dd5cfc15ec9/dependency_cache.go (about) 1 /* 2 * Copyright 2018-2020 the original author or authors. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * https://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package libpak 18 19 import ( 20 "crypto/sha256" 21 "encoding/hex" 22 "fmt" 23 "io" 24 "net" 25 "net/http" 26 "net/url" 27 "os" 28 "path/filepath" 29 "strconv" 30 "strings" 31 "time" 32 33 "github.com/BurntSushi/toml" 34 "github.com/buildpacks/libcnb" 35 "github.com/heroku/color" 36 37 "github.com/BarDweller/libpak/bard" 38 "github.com/BarDweller/libpak/sherpa" 39 ) 40 41 type HttpClientTimeouts struct { 42 DialerTimeout time.Duration 43 DialerKeepAlive time.Duration 44 TLSHandshakeTimeout time.Duration 45 ResponseHeaderTimeout time.Duration 46 ExpectContinueTimeout time.Duration 47 } 48 49 // DependencyCache allows a user to get an artifact either from a buildpack's cache, a previous download, or to download 50 // directly. 51 type DependencyCache struct { 52 53 // CachePath is the location where the buildpack has cached its dependencies. 54 CachePath string 55 56 // DownloadPath is the location of all downloads during this execution of the build. 57 DownloadPath string 58 59 // Logger is the logger used to write to the console. 60 Logger bard.Logger 61 62 // UserAgent is the User-Agent string to use with requests. 63 UserAgent string 64 65 // Mappings optionally provides URIs mapping for BuildpackDependencies 66 Mappings map[string]string 67 68 // httpClientTimeouts contains the timeout values used by HTTP client 69 HttpClientTimeouts HttpClientTimeouts 70 } 71 72 // NewDependencyCache creates a new instance setting the default cache path (<BUILDPACK_PATH>/dependencies) and user 73 // agent (<BUILDPACK_ID>/<BUILDPACK_VERSION>). 74 // Mappings will be read from any libcnb.Binding in the context with type "dependency-mappings" 75 func NewDependencyCache(context libcnb.BuildContext) (DependencyCache, error) { 76 cache := DependencyCache{ 77 CachePath: filepath.Join(context.Buildpack.Path, "dependencies"), 78 DownloadPath: os.TempDir(), 79 UserAgent: fmt.Sprintf("%s/%s", context.Buildpack.Info.ID, context.Buildpack.Info.Version), 80 Mappings: map[string]string{}, 81 } 82 mappings, err := mappingsFromBindings(context.Platform.Bindings) 83 if err != nil { 84 return DependencyCache{}, fmt.Errorf("unable to process dependency-mapping bindings\n%w", err) 85 } 86 cache.Mappings = mappings 87 88 clientTimeouts, err := customizeHttpClientTimeouts() 89 if err != nil { 90 return DependencyCache{}, fmt.Errorf("unable to read custom timeout settings\n%w", err) 91 } 92 cache.HttpClientTimeouts = *clientTimeouts 93 94 return cache, nil 95 } 96 97 func customizeHttpClientTimeouts() (*HttpClientTimeouts, error) { 98 rawStr := sherpa.GetEnvWithDefault("BP_DIALER_TIMEOUT", "6") 99 dialerTimeout, err := strconv.Atoi(rawStr) 100 if err != nil { 101 return nil, fmt.Errorf("unable to convert BP_DIALER_TIMEOUT=%s to integer\n%w", rawStr, err) 102 } 103 104 rawStr = sherpa.GetEnvWithDefault("BP_DIALER_KEEP_ALIVE", "60") 105 dialerKeepAlive, err := strconv.Atoi(rawStr) 106 if err != nil { 107 return nil, fmt.Errorf("unable to convert BP_DIALER_KEEP_ALIVE=%s to integer\n%w", rawStr, err) 108 } 109 110 rawStr = sherpa.GetEnvWithDefault("BP_TLS_HANDSHAKE_TIMEOUT", "5") 111 tlsHandshakeTimeout, err := strconv.Atoi(rawStr) 112 if err != nil { 113 return nil, fmt.Errorf("unable to convert BP_TLS_HANDSHAKE_TIMEOUT=%s to integer\n%w", rawStr, err) 114 } 115 116 rawStr = sherpa.GetEnvWithDefault("BP_RESPONSE_HEADER_TIMEOUT", "5") 117 responseHeaderTimeout, err := strconv.Atoi(rawStr) 118 if err != nil { 119 return nil, fmt.Errorf("unable to convert BP_RESPONSE_HEADER_TIMEOUT=%s to integer\n%w", rawStr, err) 120 } 121 122 rawStr = sherpa.GetEnvWithDefault("BP_EXPECT_CONTINUE_TIMEOUT", "1") 123 expectContinueTimeout, err := strconv.Atoi(rawStr) 124 if err != nil { 125 return nil, fmt.Errorf("unable to convert BP_EXPECT_CONTINUE_TIMEOUT=%s to integer\n%w", rawStr, err) 126 } 127 128 return &HttpClientTimeouts{ 129 DialerTimeout: time.Duration(dialerTimeout) * time.Second, 130 DialerKeepAlive: time.Duration(dialerKeepAlive) * time.Second, 131 TLSHandshakeTimeout: time.Duration(tlsHandshakeTimeout) * time.Second, 132 ResponseHeaderTimeout: time.Duration(responseHeaderTimeout) * time.Second, 133 ExpectContinueTimeout: time.Duration(expectContinueTimeout) * time.Second, 134 }, nil 135 } 136 137 func mappingsFromBindings(bindings libcnb.Bindings) (map[string]string, error) { 138 mappings := map[string]string{} 139 for _, binding := range bindings { 140 if strings.ToLower(binding.Type) == "dependency-mapping" { 141 for digest, uri := range binding.Secret { 142 if _, ok := mappings[digest]; ok { 143 return nil, fmt.Errorf("multiple mappings for digest %q", digest) 144 } 145 mappings[digest] = uri 146 } 147 } 148 } 149 return mappings, nil 150 } 151 152 // RequestModifierFunc is a callback that enables modification of a download request before it is sent. It is often 153 // used to set Authorization headers. 154 type RequestModifierFunc func(request *http.Request) (*http.Request, error) 155 156 // Artifact returns the path to the artifact. Resolution of that path follows three tiers: 157 // 158 // 1. CachePath 159 // 2. DownloadPath 160 // 3. Download from URI 161 // 162 // If the BuildpackDependency's SHA256 is not set, the download can never be verified to be up to date and will always 163 // download, skipping all the caches. 164 func (d *DependencyCache) Artifact(dependency BuildModuleDependency, mods ...RequestModifierFunc) (*os.File, error) { 165 166 var ( 167 actual BuildModuleDependency 168 artifact string 169 file string 170 uri = dependency.URI 171 ) 172 173 for d, u := range d.Mappings { 174 if d == dependency.SHA256 { 175 uri = u 176 break 177 } 178 } 179 180 if dependency.SHA256 == "" { 181 d.Logger.Headerf("%s Dependency has no SHA256. Skipping cache.", 182 color.New(color.FgYellow, color.Bold).Sprint("Warning:")) 183 184 d.Logger.Bodyf("%s from %s", color.YellowString("Downloading"), uri) 185 artifact = filepath.Join(d.DownloadPath, filepath.Base(uri)) 186 if err := d.download(uri, artifact, mods...); err != nil { 187 return nil, fmt.Errorf("unable to download %s\n%w", uri, err) 188 } 189 190 return os.Open(artifact) 191 } 192 193 file = filepath.Join(d.CachePath, fmt.Sprintf("%s.toml", dependency.SHA256)) 194 b, err := os.ReadFile(file) 195 if err != nil && !os.IsNotExist(err) { 196 return nil, fmt.Errorf("unable to read %s\n%w", file, err) 197 } 198 if err := toml.Unmarshal(b, &actual); err != nil { 199 return nil, fmt.Errorf("unable to decode download metadata %s\n%w", file, err) 200 } 201 202 if dependency.Equals(actual) { 203 d.Logger.Bodyf("%s cached download from buildpack", color.GreenString("Reusing")) 204 return os.Open(filepath.Join(d.CachePath, dependency.SHA256, filepath.Base(uri))) 205 } 206 207 file = filepath.Join(d.DownloadPath, fmt.Sprintf("%s.toml", dependency.SHA256)) 208 b, err = os.ReadFile(file) 209 if err != nil && !os.IsNotExist(err) { 210 return nil, fmt.Errorf("unable to read %s\n%w", file, err) 211 } 212 if err := toml.Unmarshal(b, &actual); err != nil { 213 return nil, fmt.Errorf("unable to decode download metadata %s\n%w", file, err) 214 } 215 216 if dependency.Equals(actual) { 217 d.Logger.Bodyf("%s previously cached download", color.GreenString("Reusing")) 218 return os.Open(filepath.Join(d.DownloadPath, dependency.SHA256, filepath.Base(uri))) 219 } 220 221 d.Logger.Bodyf("%s from %s", color.YellowString("Downloading"), uri) 222 artifact = filepath.Join(d.DownloadPath, dependency.SHA256, filepath.Base(uri)) 223 if err := d.download(uri, artifact, mods...); err != nil { 224 return nil, fmt.Errorf("unable to download %s\n%w", uri, err) 225 } 226 227 d.Logger.Body("Verifying checksum") 228 if err := d.verify(artifact, dependency.SHA256); err != nil { 229 return nil, err 230 } 231 232 file = filepath.Join(d.DownloadPath, fmt.Sprintf("%s.toml", dependency.SHA256)) 233 if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil { 234 return nil, fmt.Errorf("unable to make directory %s\n%w", filepath.Dir(file), err) 235 } 236 237 out, err := os.OpenFile(file, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0755) 238 if err != nil { 239 return nil, fmt.Errorf("unable to open file %s\n%w", file, err) 240 } 241 defer out.Close() 242 243 if err := toml.NewEncoder(out).Encode(dependency); err != nil { 244 return nil, fmt.Errorf("unable to write metadata %s\n%w", file, err) 245 } 246 247 return os.Open(artifact) 248 } 249 250 func (d DependencyCache) download(uri string, destination string, mods ...RequestModifierFunc) error { 251 url, err := url.Parse(uri) 252 if err != nil { 253 return fmt.Errorf("unable to parse URI %s\n%w", uri, err) 254 } 255 256 if url.Scheme == "file" { 257 return d.downloadFile(url.Path, destination, mods...) 258 } 259 260 return d.downloadHttp(uri, destination, mods...) 261 } 262 263 func (d DependencyCache) downloadFile(source string, destination string, mods ...RequestModifierFunc) error { 264 if err := os.MkdirAll(filepath.Dir(destination), 0755); err != nil { 265 return fmt.Errorf("unable to make directory %s\n%w", filepath.Dir(destination), err) 266 } 267 268 out, err := os.OpenFile(destination, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 269 if err != nil { 270 return fmt.Errorf("unable to open destination file %s\n%w", destination, err) 271 } 272 defer out.Close() 273 274 input, err := os.Open(source) 275 if err != nil { 276 return fmt.Errorf("unable to open source file %s\n%w", source, err) 277 } 278 defer out.Close() 279 280 if _, err := io.Copy(out, input); err != nil { 281 return fmt.Errorf("unable to copy from %s to %s\n%w", source, destination, err) 282 } 283 284 return nil 285 } 286 287 func (d DependencyCache) downloadHttp(uri string, destination string, mods ...RequestModifierFunc) error { 288 req, err := http.NewRequest("GET", uri, nil) 289 if err != nil { 290 return fmt.Errorf("unable to create new GET request for %s\n%w", uri, err) 291 } 292 293 if d.UserAgent != "" { 294 req.Header.Set("User-Agent", d.UserAgent) 295 } 296 297 for _, m := range mods { 298 req, err = m(req) 299 if err != nil { 300 return fmt.Errorf("unable to modify request\n%w", err) 301 } 302 } 303 304 client := http.Client{ 305 Transport: &http.Transport{ 306 Dial: (&net.Dialer{ 307 Timeout: d.HttpClientTimeouts.DialerTimeout, 308 KeepAlive: d.HttpClientTimeouts.DialerKeepAlive, 309 }).Dial, 310 TLSHandshakeTimeout: d.HttpClientTimeouts.TLSHandshakeTimeout, 311 ResponseHeaderTimeout: d.HttpClientTimeouts.ResponseHeaderTimeout, 312 ExpectContinueTimeout: d.HttpClientTimeouts.ExpectContinueTimeout, 313 Proxy: http.ProxyFromEnvironment, 314 }, 315 } 316 resp, err := client.Do(req) 317 if err != nil { 318 return fmt.Errorf("unable to request %s\n%w", uri, err) 319 } 320 defer resp.Body.Close() 321 322 if resp.StatusCode < 200 || resp.StatusCode > 299 { 323 return fmt.Errorf("could not download %s: %d", uri, resp.StatusCode) 324 } 325 326 if err := os.MkdirAll(filepath.Dir(destination), 0755); err != nil { 327 return fmt.Errorf("unable to make directory %s\n%w", filepath.Dir(destination), err) 328 } 329 330 out, err := os.OpenFile(destination, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) 331 if err != nil { 332 return fmt.Errorf("unable to open file %s\n%w", destination, err) 333 } 334 defer out.Close() 335 336 if _, err := io.Copy(out, resp.Body); err != nil { 337 return fmt.Errorf("unable to copy from %s to %s\n%w", uri, destination, err) 338 } 339 340 return nil 341 } 342 343 func (DependencyCache) verify(path string, expected string) error { 344 s := sha256.New() 345 346 in, err := os.Open(path) 347 if err != nil { 348 return fmt.Errorf("unable to verify %s\n%w", path, err) 349 } 350 defer in.Close() 351 352 if _, err := io.Copy(s, in); err != nil { 353 return fmt.Errorf("unable to read %s\n%w", path, err) 354 } 355 356 actual := hex.EncodeToString(s.Sum(nil)) 357 358 if expected != actual { 359 return fmt.Errorf("sha256 for %s %s does not match expected %s", path, actual, expected) 360 } 361 362 return nil 363 }