github.com/pachyderm/pachyderm@v1.13.4/src/server/pkg/storage/fileset/index/util.go (about)

     1  package index
     2  
     3  import (
     4  	"sort"
     5  
     6  	"github.com/pachyderm/pachyderm/src/server/pkg/storage/chunk"
     7  )
     8  
     9  // Perm calls f with each permutation of a.
    10  func Perm(a []rune, f func([]rune)) {
    11  	perm(a, f, 0)
    12  }
    13  
    14  // Permute the values at index i to len(a)-1.
    15  func perm(a []rune, f func([]rune), i int) {
    16  	if i > len(a) {
    17  		f(a)
    18  		return
    19  	}
    20  	perm(a, f, i+1)
    21  	for j := i + 1; j < len(a); j++ {
    22  		a[i], a[j] = a[j], a[i]
    23  		perm(a, f, i+1)
    24  		a[i], a[j] = a[j], a[i]
    25  	}
    26  }
    27  
    28  // Generate generates the permutations of the passed in string and returns them sorted.
    29  func Generate(s string) []string {
    30  	fileNames := []string{}
    31  	Perm([]rune(s), func(fileName []rune) {
    32  		fileNames = append(fileNames, string(fileName))
    33  	})
    34  	sort.Strings(fileNames)
    35  	return fileNames
    36  }
    37  
    38  // PointsTo returns a list of all the chunks this index references
    39  func PointsTo(idx *Index) []chunk.ID {
    40  	if idx == nil {
    41  		return nil
    42  	}
    43  	if idx.Range != nil {
    44  		return []chunk.ID{chunk.ID(idx.Range.ChunkRef.Ref.Id)}
    45  	}
    46  	var ids []chunk.ID
    47  	if idx.File != nil {
    48  		for _, dr := range idx.File.DataRefs {
    49  			ids = append(ids, chunk.ID(dr.Ref.Id))
    50  		}
    51  	}
    52  	return ids
    53  }
    54  
    55  // SizeBytes computes the size of the indexed data in bytes.
    56  func SizeBytes(idx *Index) int64 {
    57  	var size int64
    58  	if idx == nil {
    59  		return size
    60  	}
    61  	if idx.File != nil {
    62  		if idx.File.DataRefs != nil {
    63  			for _, dataRef := range idx.File.DataRefs {
    64  				size += dataRef.SizeBytes
    65  			}
    66  			return size
    67  		}
    68  		for _, part := range idx.File.Parts {
    69  			for _, dataRef := range part.DataRefs {
    70  				size += dataRef.SizeBytes
    71  			}
    72  		}
    73  	}
    74  	return size
    75  }
    76  
    77  // TODO: Change this such that it returns a new index with the updated fields, rather than updating in place.
    78  func resolveParts(idx *Index) {
    79  	if idx.File.DataRefs == nil {
    80  		return
    81  	}
    82  	dataRefs := idx.File.DataRefs
    83  	offset := dataRefs[0].OffsetBytes
    84  	size := dataRefs[0].SizeBytes
    85  	for _, part := range idx.File.Parts {
    86  		bytesLeft := part.SizeBytes
    87  		for size <= bytesLeft {
    88  			part.DataRefs = append(part.DataRefs, newDataRef(dataRefs[0].Ref, offset, size))
    89  			bytesLeft -= size
    90  			dataRefs = dataRefs[1:]
    91  			if len(dataRefs) == 0 {
    92  				return
    93  			}
    94  			offset = dataRefs[0].OffsetBytes
    95  			size = dataRefs[0].SizeBytes
    96  		}
    97  		part.DataRefs = append(part.DataRefs, newDataRef(dataRefs[0].Ref, offset, bytesLeft))
    98  		offset += bytesLeft
    99  		size -= bytesLeft
   100  	}
   101  }
   102  
   103  func newDataRef(chunkRef *chunk.Ref, offset, size int64) *chunk.DataRef {
   104  	return &chunk.DataRef{
   105  		Ref:         chunkRef,
   106  		OffsetBytes: offset,
   107  		SizeBytes:   size,
   108  	}
   109  }
   110  
   111  func unresolveParts(idx *Index) {
   112  	for _, part := range idx.File.Parts {
   113  		part.DataRefs = nil
   114  	}
   115  }