github.com/GoogleContainerTools/skaffold@v1.39.18/pkg/skaffold/build/cache/hash.go (about)

     1  /*
     2  Copyright 2019 The Skaffold Authors
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package cache
    18  
    19  import (
    20  	"context"
    21  	"crypto/md5"
    22  	"crypto/sha256"
    23  	"encoding/hex"
    24  	"encoding/json"
    25  	"fmt"
    26  	"io"
    27  	"os"
    28  	"sort"
    29  
    30  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/buildpacks"
    31  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
    32  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
    33  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph"
    34  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/instrumentation"
    35  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/output/log"
    36  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/platform"
    37  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
    38  	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
    39  )
    40  
    41  // For testing
    42  var (
    43  	newArtifactHasherFunc = newArtifactHasher
    44  	fileHasherFunc        = fileHasher
    45  	artifactConfigFunc    = artifactConfig
    46  )
    47  
    48  type artifactHasher interface {
    49  	hash(context.Context, *latest.Artifact, platform.Resolver) (string, error)
    50  }
    51  
    52  type artifactHasherImpl struct {
    53  	artifacts graph.ArtifactGraph
    54  	lister    DependencyLister
    55  	mode      config.RunMode
    56  	syncStore *util.SyncStore
    57  }
    58  
    59  // newArtifactHasher returns a new instance of an artifactHasher. Use newArtifactHasherFunc instead of calling this function directly.
    60  func newArtifactHasher(artifacts graph.ArtifactGraph, lister DependencyLister, mode config.RunMode) artifactHasher {
    61  	return &artifactHasherImpl{
    62  		artifacts: artifacts,
    63  		lister:    lister,
    64  		mode:      mode,
    65  		syncStore: util.NewSyncStore(),
    66  	}
    67  }
    68  
    69  func (h *artifactHasherImpl) hash(ctx context.Context, a *latest.Artifact, platforms platform.Resolver) (string, error) {
    70  	ctx, endTrace := instrumentation.StartTrace(ctx, "hash_GenerateHashOneArtifact", map[string]string{
    71  		"ImageName": instrumentation.PII(a.ImageName),
    72  	})
    73  	defer endTrace()
    74  
    75  	hash, err := h.safeHash(ctx, a, platforms.GetPlatforms(a.ImageName))
    76  	if err != nil {
    77  		endTrace(instrumentation.TraceEndError(err))
    78  		return "", err
    79  	}
    80  	hashes := []string{hash}
    81  	for _, dep := range sortedDependencies(a, h.artifacts) {
    82  		depHash, err := h.hash(ctx, dep, platforms)
    83  		if err != nil {
    84  			endTrace(instrumentation.TraceEndError(err))
    85  			return "", err
    86  		}
    87  		hashes = append(hashes, depHash)
    88  	}
    89  
    90  	if len(hashes) == 1 {
    91  		return hashes[0], nil
    92  	}
    93  	return encode(hashes)
    94  }
    95  
    96  func (h *artifactHasherImpl) safeHash(ctx context.Context, a *latest.Artifact, platforms platform.Matcher) (string, error) {
    97  	val := h.syncStore.Exec(a.ImageName,
    98  		func() interface{} {
    99  			hash, err := singleArtifactHash(ctx, h.lister, a, h.mode, platforms)
   100  			if err != nil {
   101  				return err
   102  			}
   103  			return hash
   104  		})
   105  	switch t := val.(type) {
   106  	case error:
   107  		return "", t
   108  	case string:
   109  		return t, nil
   110  	default:
   111  		return "", fmt.Errorf("internal error when retrieving cache result of type %T", t)
   112  	}
   113  }
   114  
   115  // singleArtifactHash calculates the hash for a single artifact, and ignores its required artifacts.
   116  func singleArtifactHash(ctx context.Context, depLister DependencyLister, a *latest.Artifact, mode config.RunMode, m platform.Matcher) (string, error) {
   117  	var inputs []string
   118  
   119  	// Append the artifact's configuration
   120  	config, err := artifactConfigFunc(a)
   121  	if err != nil {
   122  		return "", fmt.Errorf("getting artifact's configuration for %q: %w", a.ImageName, err)
   123  	}
   124  	inputs = append(inputs, config)
   125  
   126  	// Append the digest of each input file
   127  	deps, err := depLister(ctx, a)
   128  	if err != nil {
   129  		return "", fmt.Errorf("getting dependencies for %q: %w", a.ImageName, err)
   130  	}
   131  	sort.Strings(deps)
   132  
   133  	for _, d := range deps {
   134  		h, err := fileHasherFunc(d)
   135  		if err != nil {
   136  			if os.IsNotExist(err) {
   137  				log.Entry(ctx).Tracef("skipping dependency for artifact cache calculation, file not found %s: %s", d, err)
   138  				continue // Ignore files that don't exist
   139  			}
   140  
   141  			return "", fmt.Errorf("getting hash for %q: %w", d, err)
   142  		}
   143  		inputs = append(inputs, h)
   144  	}
   145  
   146  	// add build args for the artifact if specified
   147  	args, err := hashBuildArgs(a, mode)
   148  	if err != nil {
   149  		return "", fmt.Errorf("hashing build args: %w", err)
   150  	}
   151  	if args != nil {
   152  		inputs = append(inputs, args...)
   153  	}
   154  
   155  	// add build platforms
   156  	var ps []string
   157  	for _, p := range m.Platforms {
   158  		ps = append(ps, platform.Format(p))
   159  	}
   160  	sort.Strings(ps)
   161  	inputs = append(inputs, ps...)
   162  
   163  	return encode(inputs)
   164  }
   165  
   166  func encode(inputs []string) (string, error) {
   167  	// get a key for the hashes
   168  	hasher := sha256.New()
   169  	enc := json.NewEncoder(hasher)
   170  	if err := enc.Encode(inputs); err != nil {
   171  		return "", err
   172  	}
   173  	return hex.EncodeToString(hasher.Sum(nil)), nil
   174  }
   175  
   176  // TODO(dgageot): when the buildpacks builder image digest changes, we need to change the hash
   177  func artifactConfig(a *latest.Artifact) (string, error) {
   178  	buf, err := json.Marshal(a.ArtifactType)
   179  	if err != nil {
   180  		return "", fmt.Errorf("marshalling the artifact's configuration for %q: %w", a.ImageName, err)
   181  	}
   182  	return string(buf), nil
   183  }
   184  
   185  func hashBuildArgs(artifact *latest.Artifact, mode config.RunMode) ([]string, error) {
   186  	// only one of args or env is ever populated
   187  	var args map[string]*string
   188  	var env map[string]string
   189  	var err error
   190  	switch {
   191  	case artifact.DockerArtifact != nil:
   192  		args, err = docker.EvalBuildArgs(mode, artifact.Workspace, artifact.DockerArtifact.DockerfilePath, artifact.DockerArtifact.BuildArgs, nil)
   193  	case artifact.KanikoArtifact != nil:
   194  		args, err = docker.EvalBuildArgs(mode, artifact.Workspace, artifact.KanikoArtifact.DockerfilePath, artifact.KanikoArtifact.BuildArgs, nil)
   195  	case artifact.BuildpackArtifact != nil:
   196  		env, err = buildpacks.GetEnv(artifact, mode)
   197  	case artifact.CustomArtifact != nil && artifact.CustomArtifact.Dependencies.Dockerfile != nil:
   198  		args, err = util.EvaluateEnvTemplateMap(artifact.CustomArtifact.Dependencies.Dockerfile.BuildArgs)
   199  	default:
   200  		return nil, nil
   201  	}
   202  	if err != nil {
   203  		return nil, err
   204  	}
   205  	var sl []string
   206  	if args != nil {
   207  		sl = util.EnvPtrMapToSlice(args, "=")
   208  	}
   209  	if env != nil {
   210  		sl = util.EnvMapToSlice(env, "=")
   211  	}
   212  	return sl, nil
   213  }
   214  
   215  // fileHasher hashes the contents and name of a file
   216  func fileHasher(p string) (string, error) {
   217  	h := md5.New()
   218  	fi, err := os.Lstat(p)
   219  	if err != nil {
   220  		return "", err
   221  	}
   222  	h.Write([]byte(fi.Mode().String()))
   223  	h.Write([]byte(fi.Name()))
   224  	if fi.Mode().IsRegular() {
   225  		f, err := os.Open(p)
   226  		if err != nil {
   227  			return "", err
   228  		}
   229  		defer f.Close()
   230  		if _, err := io.Copy(h, f); err != nil {
   231  			return "", err
   232  		}
   233  	}
   234  	return hex.EncodeToString(h.Sum(nil)), nil
   235  }
   236  
   237  // sortedDependencies returns the dependencies' corresponding Artifacts as sorted by their image name.
   238  func sortedDependencies(a *latest.Artifact, artifacts graph.ArtifactGraph) []*latest.Artifact {
   239  	sl := artifacts.Dependencies(a)
   240  	sort.Slice(sl, func(i, j int) bool {
   241  		ia, ja := sl[i], sl[j]
   242  		return ia.ImageName < ja.ImageName
   243  	})
   244  	return sl
   245  }