github.com/stffabi/git-lfs@v2.3.5-0.20180214015214-8eeaa8d88902+incompatible/commands/command_pull.go (about)

     1  package commands
     2  
     3  import (
     4  	"fmt"
     5  	"os"
     6  	"sync"
     7  	"time"
     8  
     9  	"github.com/git-lfs/git-lfs/filepathfilter"
    10  	"github.com/git-lfs/git-lfs/git"
    11  	"github.com/git-lfs/git-lfs/lfs"
    12  	"github.com/git-lfs/git-lfs/tasklog"
    13  	"github.com/git-lfs/git-lfs/tq"
    14  	"github.com/rubyist/tracerx"
    15  	"github.com/spf13/cobra"
    16  )
    17  
    18  func pullCommand(cmd *cobra.Command, args []string) {
    19  	requireGitVersion()
    20  	requireInRepo()
    21  
    22  	if len(args) > 0 {
    23  		// Remote is first arg
    24  		if err := cfg.SetValidRemote(args[0]); err != nil {
    25  			Exit("Invalid remote name %q: %s", args[0], err)
    26  		}
    27  	}
    28  
    29  	includeArg, excludeArg := getIncludeExcludeArgs(cmd)
    30  	filter := buildFilepathFilter(cfg, includeArg, excludeArg)
    31  	pull(filter)
    32  }
    33  
    34  func pull(filter *filepathfilter.Filter) {
    35  	ref, err := git.CurrentRef()
    36  	if err != nil {
    37  		Panic(err, "Could not pull")
    38  	}
    39  
    40  	pointers := newPointerMap()
    41  	logger := tasklog.NewLogger(os.Stdout)
    42  	meter := tq.NewMeter()
    43  	meter.Logger = meter.LoggerFromEnv(cfg.Os)
    44  	logger.Enqueue(meter)
    45  	remote := cfg.Remote()
    46  	singleCheckout := newSingleCheckout(cfg.Git, remote)
    47  	q := newDownloadQueue(singleCheckout.Manifest(), remote, tq.WithProgress(meter))
    48  	gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
    49  		if err != nil {
    50  			LoggedError(err, "Scanner error: %s", err)
    51  			return
    52  		}
    53  
    54  		if pointers.Seen(p) {
    55  			return
    56  		}
    57  
    58  		// no need to download objects that exist locally already
    59  		lfs.LinkOrCopyFromReference(cfg, p.Oid, p.Size)
    60  		if cfg.LFSObjectExists(p.Oid, p.Size) {
    61  			singleCheckout.Run(p)
    62  			return
    63  		}
    64  
    65  		meter.Add(p.Size)
    66  		tracerx.Printf("fetch %v [%v]", p.Name, p.Oid)
    67  		pointers.Add(p)
    68  		q.Add(downloadTransfer(p))
    69  	})
    70  
    71  	gitscanner.Filter = filter
    72  
    73  	dlwatch := q.Watch()
    74  	var wg sync.WaitGroup
    75  	wg.Add(1)
    76  
    77  	go func() {
    78  		for t := range dlwatch {
    79  			for _, p := range pointers.All(t.Oid) {
    80  				singleCheckout.Run(p)
    81  			}
    82  		}
    83  		wg.Done()
    84  	}()
    85  
    86  	processQueue := time.Now()
    87  	if err := gitscanner.ScanTree(ref.Sha); err != nil {
    88  		singleCheckout.Close()
    89  		ExitWithError(err)
    90  	}
    91  
    92  	meter.Start()
    93  	gitscanner.Close()
    94  	q.Wait()
    95  	wg.Wait()
    96  	tracerx.PerformanceSince("process queue", processQueue)
    97  
    98  	singleCheckout.Close()
    99  
   100  	success := true
   101  	for _, err := range q.Errors() {
   102  		success = false
   103  		FullError(err)
   104  	}
   105  
   106  	if !success {
   107  		c := getAPIClient()
   108  		e := c.Endpoints.Endpoint("download", remote)
   109  		Exit("error: failed to fetch some objects from '%s'", e.Url)
   110  	}
   111  
   112  	if singleCheckout.Skip() {
   113  		fmt.Println("Skipping object checkout, Git LFS is not installed.")
   114  	}
   115  }
   116  
   117  // tracks LFS objects being downloaded, according to their unique OIDs.
   118  type pointerMap struct {
   119  	pointers map[string][]*lfs.WrappedPointer
   120  	mu       sync.Mutex
   121  }
   122  
   123  func newPointerMap() *pointerMap {
   124  	return &pointerMap{pointers: make(map[string][]*lfs.WrappedPointer)}
   125  }
   126  
   127  func (m *pointerMap) Seen(p *lfs.WrappedPointer) bool {
   128  	m.mu.Lock()
   129  	defer m.mu.Unlock()
   130  	if existing, ok := m.pointers[p.Oid]; ok {
   131  		m.pointers[p.Oid] = append(existing, p)
   132  		return true
   133  	}
   134  	return false
   135  }
   136  
   137  func (m *pointerMap) Add(p *lfs.WrappedPointer) {
   138  	m.mu.Lock()
   139  	defer m.mu.Unlock()
   140  	m.pointers[p.Oid] = append(m.pointers[p.Oid], p)
   141  }
   142  
   143  func (m *pointerMap) All(oid string) []*lfs.WrappedPointer {
   144  	m.mu.Lock()
   145  	defer m.mu.Unlock()
   146  	pointers := m.pointers[oid]
   147  	delete(m.pointers, oid)
   148  	return pointers
   149  }
   150  
   151  func init() {
   152  	RegisterCommand("pull", pullCommand, func(cmd *cobra.Command) {
   153  		cmd.Flags().StringVarP(&includeArg, "include", "I", "", "Include a list of paths")
   154  		cmd.Flags().StringVarP(&excludeArg, "exclude", "X", "", "Exclude a list of paths")
   155  	})
   156  }