github.com/2lambda123/git-lfs@v2.5.2+incompatible/lfs/scanner.go (about)

     1  package lfs
     2  
     3  import "github.com/git-lfs/git-lfs/tools"
     4  
     5  const (
     6  	// blobSizeCutoff is used to determine which files to scan for Git LFS
     7  	// pointers.  Any file with a size below this cutoff will be scanned.
     8  	blobSizeCutoff = 1024
     9  
    10  	// stdoutBufSize is the size of the buffers given to a sub-process stdout
    11  	stdoutBufSize = 16384
    12  
    13  	// chanBufSize is the size of the channels used to pass data from one
    14  	// sub-process to another.
    15  	chanBufSize = 100
    16  )
    17  
    18  // WrappedPointer wraps a pointer.Pointer and provides the git sha1
    19  // and the file name associated with the object, taken from the
    20  // rev-list output.
    21  type WrappedPointer struct {
    22  	Sha1    string
    23  	Name    string
    24  	SrcName string
    25  	Status  string
    26  	*Pointer
    27  }
    28  
    29  // catFileBatchCheck uses git cat-file --batch-check to get the type
    30  // and size of a git object. Any object that isn't of type blob and
    31  // under the blobSizeCutoff will be ignored. revs is a channel over
    32  // which strings containing git sha1s will be sent. It returns a channel
    33  // from which sha1 strings can be read.
    34  func catFileBatchCheck(revs *StringChannelWrapper, lockableSet *lockableNameSet) (*StringChannelWrapper, chan string, error) {
    35  	smallRevCh := make(chan string, chanBufSize)
    36  	lockableCh := make(chan string, chanBufSize)
    37  	errCh := make(chan error, 2) // up to 2 errors, one from each goroutine
    38  	if err := runCatFileBatchCheck(smallRevCh, lockableCh, lockableSet, revs, errCh); err != nil {
    39  		return nil, nil, err
    40  	}
    41  	return NewStringChannelWrapper(smallRevCh, errCh), lockableCh, nil
    42  }
    43  
    44  // catFileBatch uses git cat-file --batch to get the object contents
    45  // of a git object, given its sha1. The contents will be decoded into
    46  // a Git LFS pointer. revs is a channel over which strings containing Git SHA1s
    47  // will be sent. It returns a channel from which point.Pointers can be read.
    48  func catFileBatch(revs *StringChannelWrapper, lockableSet *lockableNameSet) (*PointerChannelWrapper, chan string, error) {
    49  	pointerCh := make(chan *WrappedPointer, chanBufSize)
    50  	lockableCh := make(chan string, chanBufSize)
    51  	errCh := make(chan error, 5) // shared by 2 goroutines & may add more detail errors?
    52  	if err := runCatFileBatch(pointerCh, lockableCh, lockableSet, revs, errCh); err != nil {
    53  		return nil, nil, err
    54  	}
    55  	return NewPointerChannelWrapper(pointerCh, errCh), lockableCh, nil
    56  }
    57  
    58  // ChannelWrapper for pointer Scan* functions to more easily return async error data via Wait()
    59  // See NewPointerChannelWrapper for construction / use
    60  type PointerChannelWrapper struct {
    61  	*tools.BaseChannelWrapper
    62  	Results <-chan *WrappedPointer
    63  }
    64  
    65  // Construct a new channel wrapper for WrappedPointer
    66  // Caller can use s.Results directly for normal processing then call Wait() to finish & check for errors
    67  // Scan function is required to create error channel large enough not to block (usually 1 is ok)
    68  func NewPointerChannelWrapper(pointerChan <-chan *WrappedPointer, errorChan <-chan error) *PointerChannelWrapper {
    69  	return &PointerChannelWrapper{tools.NewBaseChannelWrapper(errorChan), pointerChan}
    70  }
    71  
    72  // ChannelWrapper for string channel functions to more easily return async error data via Wait()
    73  // Caller can use s.Results directly for normal processing then call Wait() to finish & check for errors
    74  // See NewStringChannelWrapper for construction / use
    75  type StringChannelWrapper struct {
    76  	*tools.BaseChannelWrapper
    77  	Results <-chan string
    78  }
    79  
    80  // Construct a new channel wrapper for string
    81  // Caller can use s.Results directly for normal processing then call Wait() to finish & check for errors
    82  func NewStringChannelWrapper(stringChan <-chan string, errorChan <-chan error) *StringChannelWrapper {
    83  	return &StringChannelWrapper{tools.NewBaseChannelWrapper(errorChan), stringChan}
    84  }
    85  
    86  // ChannelWrapper for TreeBlob channel functions to more easily return async error data via Wait()
    87  // See NewTreeBlobChannelWrapper for construction / use
    88  type TreeBlobChannelWrapper struct {
    89  	*tools.BaseChannelWrapper
    90  	Results <-chan TreeBlob
    91  }
    92  
    93  // Construct a new channel wrapper for TreeBlob
    94  // Caller can use s.Results directly for normal processing then call Wait() to finish & check for errors
    95  func NewTreeBlobChannelWrapper(treeBlobChan <-chan TreeBlob, errorChan <-chan error) *TreeBlobChannelWrapper {
    96  	return &TreeBlobChannelWrapper{tools.NewBaseChannelWrapper(errorChan), treeBlobChan}
    97  }