github.com/GoogleContainerTools/skaffold@v1.39.18/pkg/skaffold/build/cache/hash.go (about) 1 /* 2 Copyright 2019 The Skaffold Authors 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package cache 18 19 import ( 20 "context" 21 "crypto/md5" 22 "crypto/sha256" 23 "encoding/hex" 24 "encoding/json" 25 "fmt" 26 "io" 27 "os" 28 "sort" 29 30 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/buildpacks" 31 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/config" 32 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker" 33 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph" 34 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/instrumentation" 35 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/output/log" 36 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/platform" 37 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" 38 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util" 39 ) 40 41 // For testing 42 var ( 43 newArtifactHasherFunc = newArtifactHasher 44 fileHasherFunc = fileHasher 45 artifactConfigFunc = artifactConfig 46 ) 47 48 type artifactHasher interface { 49 hash(context.Context, *latest.Artifact, platform.Resolver) (string, error) 50 } 51 52 type artifactHasherImpl struct { 53 artifacts graph.ArtifactGraph 54 lister DependencyLister 55 mode config.RunMode 56 syncStore *util.SyncStore 57 } 58 59 // newArtifactHasher returns a new instance of an artifactHasher. Use newArtifactHasherFunc instead of calling this function directly. 60 func newArtifactHasher(artifacts graph.ArtifactGraph, lister DependencyLister, mode config.RunMode) artifactHasher { 61 return &artifactHasherImpl{ 62 artifacts: artifacts, 63 lister: lister, 64 mode: mode, 65 syncStore: util.NewSyncStore(), 66 } 67 } 68 69 func (h *artifactHasherImpl) hash(ctx context.Context, a *latest.Artifact, platforms platform.Resolver) (string, error) { 70 ctx, endTrace := instrumentation.StartTrace(ctx, "hash_GenerateHashOneArtifact", map[string]string{ 71 "ImageName": instrumentation.PII(a.ImageName), 72 }) 73 defer endTrace() 74 75 hash, err := h.safeHash(ctx, a, platforms.GetPlatforms(a.ImageName)) 76 if err != nil { 77 endTrace(instrumentation.TraceEndError(err)) 78 return "", err 79 } 80 hashes := []string{hash} 81 for _, dep := range sortedDependencies(a, h.artifacts) { 82 depHash, err := h.hash(ctx, dep, platforms) 83 if err != nil { 84 endTrace(instrumentation.TraceEndError(err)) 85 return "", err 86 } 87 hashes = append(hashes, depHash) 88 } 89 90 if len(hashes) == 1 { 91 return hashes[0], nil 92 } 93 return encode(hashes) 94 } 95 96 func (h *artifactHasherImpl) safeHash(ctx context.Context, a *latest.Artifact, platforms platform.Matcher) (string, error) { 97 val := h.syncStore.Exec(a.ImageName, 98 func() interface{} { 99 hash, err := singleArtifactHash(ctx, h.lister, a, h.mode, platforms) 100 if err != nil { 101 return err 102 } 103 return hash 104 }) 105 switch t := val.(type) { 106 case error: 107 return "", t 108 case string: 109 return t, nil 110 default: 111 return "", fmt.Errorf("internal error when retrieving cache result of type %T", t) 112 } 113 } 114 115 // singleArtifactHash calculates the hash for a single artifact, and ignores its required artifacts. 116 func singleArtifactHash(ctx context.Context, depLister DependencyLister, a *latest.Artifact, mode config.RunMode, m platform.Matcher) (string, error) { 117 var inputs []string 118 119 // Append the artifact's configuration 120 config, err := artifactConfigFunc(a) 121 if err != nil { 122 return "", fmt.Errorf("getting artifact's configuration for %q: %w", a.ImageName, err) 123 } 124 inputs = append(inputs, config) 125 126 // Append the digest of each input file 127 deps, err := depLister(ctx, a) 128 if err != nil { 129 return "", fmt.Errorf("getting dependencies for %q: %w", a.ImageName, err) 130 } 131 sort.Strings(deps) 132 133 for _, d := range deps { 134 h, err := fileHasherFunc(d) 135 if err != nil { 136 if os.IsNotExist(err) { 137 log.Entry(ctx).Tracef("skipping dependency for artifact cache calculation, file not found %s: %s", d, err) 138 continue // Ignore files that don't exist 139 } 140 141 return "", fmt.Errorf("getting hash for %q: %w", d, err) 142 } 143 inputs = append(inputs, h) 144 } 145 146 // add build args for the artifact if specified 147 args, err := hashBuildArgs(a, mode) 148 if err != nil { 149 return "", fmt.Errorf("hashing build args: %w", err) 150 } 151 if args != nil { 152 inputs = append(inputs, args...) 153 } 154 155 // add build platforms 156 var ps []string 157 for _, p := range m.Platforms { 158 ps = append(ps, platform.Format(p)) 159 } 160 sort.Strings(ps) 161 inputs = append(inputs, ps...) 162 163 return encode(inputs) 164 } 165 166 func encode(inputs []string) (string, error) { 167 // get a key for the hashes 168 hasher := sha256.New() 169 enc := json.NewEncoder(hasher) 170 if err := enc.Encode(inputs); err != nil { 171 return "", err 172 } 173 return hex.EncodeToString(hasher.Sum(nil)), nil 174 } 175 176 // TODO(dgageot): when the buildpacks builder image digest changes, we need to change the hash 177 func artifactConfig(a *latest.Artifact) (string, error) { 178 buf, err := json.Marshal(a.ArtifactType) 179 if err != nil { 180 return "", fmt.Errorf("marshalling the artifact's configuration for %q: %w", a.ImageName, err) 181 } 182 return string(buf), nil 183 } 184 185 func hashBuildArgs(artifact *latest.Artifact, mode config.RunMode) ([]string, error) { 186 // only one of args or env is ever populated 187 var args map[string]*string 188 var env map[string]string 189 var err error 190 switch { 191 case artifact.DockerArtifact != nil: 192 args, err = docker.EvalBuildArgs(mode, artifact.Workspace, artifact.DockerArtifact.DockerfilePath, artifact.DockerArtifact.BuildArgs, nil) 193 case artifact.KanikoArtifact != nil: 194 args, err = docker.EvalBuildArgs(mode, artifact.Workspace, artifact.KanikoArtifact.DockerfilePath, artifact.KanikoArtifact.BuildArgs, nil) 195 case artifact.BuildpackArtifact != nil: 196 env, err = buildpacks.GetEnv(artifact, mode) 197 case artifact.CustomArtifact != nil && artifact.CustomArtifact.Dependencies.Dockerfile != nil: 198 args, err = util.EvaluateEnvTemplateMap(artifact.CustomArtifact.Dependencies.Dockerfile.BuildArgs) 199 default: 200 return nil, nil 201 } 202 if err != nil { 203 return nil, err 204 } 205 var sl []string 206 if args != nil { 207 sl = util.EnvPtrMapToSlice(args, "=") 208 } 209 if env != nil { 210 sl = util.EnvMapToSlice(env, "=") 211 } 212 return sl, nil 213 } 214 215 // fileHasher hashes the contents and name of a file 216 func fileHasher(p string) (string, error) { 217 h := md5.New() 218 fi, err := os.Lstat(p) 219 if err != nil { 220 return "", err 221 } 222 h.Write([]byte(fi.Mode().String())) 223 h.Write([]byte(fi.Name())) 224 if fi.Mode().IsRegular() { 225 f, err := os.Open(p) 226 if err != nil { 227 return "", err 228 } 229 defer f.Close() 230 if _, err := io.Copy(h, f); err != nil { 231 return "", err 232 } 233 } 234 return hex.EncodeToString(h.Sum(nil)), nil 235 } 236 237 // sortedDependencies returns the dependencies' corresponding Artifacts as sorted by their image name. 238 func sortedDependencies(a *latest.Artifact, artifacts graph.ArtifactGraph) []*latest.Artifact { 239 sl := artifacts.Dependencies(a) 240 sort.Slice(sl, func(i, j int) bool { 241 ia, ja := sl[i], sl[j] 242 return ia.ImageName < ja.ImageName 243 }) 244 return sl 245 }