github.com/pachyderm/pachyderm@v1.13.4/src/server/worker/common/common.go (about)

     1  package common
     2  
     3  import (
     4  	"context"
     5  	"crypto/sha256"
     6  	"encoding/base64"
     7  	"encoding/hex"
     8  
     9  	"github.com/pachyderm/pachyderm/src/client"
    10  	"github.com/pachyderm/pachyderm/src/client/pps"
    11  )
    12  
    13  // IsDone returns true if the given context has been canceled, or false otherwise
    14  func IsDone(ctx context.Context) bool {
    15  	select {
    16  	case <-ctx.Done():
    17  		return true
    18  	default:
    19  		return false
    20  	}
    21  }
    22  
    23  // DatumID computes the id for a datum, this value is used in ListDatum and
    24  // InspectDatum.
    25  func DatumID(inputs []*Input) string {
    26  	hash := sha256.New()
    27  	for _, input := range inputs {
    28  		hash.Write([]byte(input.FileInfo.File.Path))
    29  		hash.Write(input.FileInfo.Hash)
    30  	}
    31  	// InputFileID is a single string id for the data from this input, it's used in logs and in
    32  	// the statsTree
    33  	return hex.EncodeToString(hash.Sum(nil))
    34  }
    35  
    36  // HashDatum computes and returns the hash of datum + pipeline, with a
    37  // pipeline-specific prefix.
    38  func HashDatum(pipelineName string, pipelineSalt string, inputs []*Input) string {
    39  	hash := sha256.New()
    40  	for _, input := range inputs {
    41  		hash.Write([]byte(input.Name))
    42  		hash.Write([]byte(input.FileInfo.File.Path))
    43  		hash.Write(input.FileInfo.Hash)
    44  	}
    45  
    46  	hash.Write([]byte(pipelineName))
    47  	hash.Write([]byte(pipelineSalt))
    48  
    49  	return client.DatumTagPrefix(pipelineSalt) + hex.EncodeToString(hash.Sum(nil))
    50  }
    51  
    52  // MatchDatum checks if a datum matches a filter.  To match each string in
    53  // filter must correspond match at least 1 datum's Path or Hash. Order of
    54  // filter and inputs is irrelevant.
    55  func MatchDatum(filter []string, inputs []*pps.InputFile) bool {
    56  	// All paths in request.DataFilters must appear somewhere in the log
    57  	// line's inputs, or it's filtered
    58  	matchesData := true
    59  dataFilters:
    60  	for _, dataFilter := range filter {
    61  		for _, input := range inputs {
    62  			if dataFilter == input.Path ||
    63  				dataFilter == base64.StdEncoding.EncodeToString(input.Hash) ||
    64  				dataFilter == hex.EncodeToString(input.Hash) {
    65  				continue dataFilters // Found, move to next filter
    66  			}
    67  		}
    68  		matchesData = false
    69  		break
    70  	}
    71  	return matchesData
    72  }