github.com/psexton/git-lfs@v2.1.1-0.20170517224304-289a18b2bc53+incompatible/commands/command_fetch.go (about) 1 package commands 2 3 import ( 4 "fmt" 5 "time" 6 7 "github.com/git-lfs/git-lfs/filepathfilter" 8 "github.com/git-lfs/git-lfs/git" 9 "github.com/git-lfs/git-lfs/lfs" 10 "github.com/git-lfs/git-lfs/progress" 11 "github.com/git-lfs/git-lfs/tq" 12 "github.com/rubyist/tracerx" 13 "github.com/spf13/cobra" 14 ) 15 16 var ( 17 fetchRecentArg bool 18 fetchAllArg bool 19 fetchPruneArg bool 20 ) 21 22 func getIncludeExcludeArgs(cmd *cobra.Command) (include, exclude *string) { 23 includeFlag := cmd.Flag("include") 24 excludeFlag := cmd.Flag("exclude") 25 if includeFlag.Changed { 26 include = &includeArg 27 } 28 if excludeFlag.Changed { 29 exclude = &excludeArg 30 } 31 32 return 33 } 34 35 func fetchCommand(cmd *cobra.Command, args []string) { 36 requireInRepo() 37 38 var refs []*git.Ref 39 40 if len(args) > 0 { 41 // Remote is first arg 42 if err := git.ValidateRemote(args[0]); err != nil { 43 Exit("Invalid remote name %q", args[0]) 44 } 45 cfg.CurrentRemote = args[0] 46 } else { 47 cfg.CurrentRemote = "" 48 } 49 50 if len(args) > 1 { 51 resolvedrefs, err := git.ResolveRefs(args[1:]) 52 if err != nil { 53 Panic(err, "Invalid ref argument: %v", args[1:]) 54 } 55 refs = resolvedrefs 56 } else if !fetchAllArg { 57 ref, err := git.CurrentRef() 58 if err != nil { 59 Panic(err, "Could not fetch") 60 } 61 refs = []*git.Ref{ref} 62 } 63 64 success := true 65 gitscanner := lfs.NewGitScanner(nil) 66 defer gitscanner.Close() 67 68 include, exclude := getIncludeExcludeArgs(cmd) 69 70 if fetchAllArg { 71 if fetchRecentArg || len(args) > 1 { 72 Exit("Cannot combine --all with ref arguments or --recent") 73 } 74 if include != nil || exclude != nil { 75 Exit("Cannot combine --all with --include or --exclude") 76 } 77 if len(cfg.FetchIncludePaths()) > 0 || len(cfg.FetchExcludePaths()) > 0 { 78 Print("Ignoring global include / exclude paths to fulfil --all") 79 } 80 success = fetchAll() 81 82 } else { // !all 83 filter := buildFilepathFilter(cfg, include, exclude) 84 85 // Fetch refs sequentially per arg order; duplicates in later refs will be ignored 86 for _, ref := range refs { 87 Print("Fetching %v", ref.Name) 88 s := fetchRef(ref.Sha, filter) 89 success = success && s 90 } 91 92 if fetchRecentArg || cfg.FetchPruneConfig().FetchRecentAlways { 93 s := fetchRecent(refs, filter) 94 success = success && s 95 } 96 } 97 98 if fetchPruneArg { 99 fetchconf := cfg.FetchPruneConfig() 100 verify := fetchconf.PruneVerifyRemoteAlways 101 // no dry-run or verbose options in fetch, assume false 102 prune(fetchconf, verify, false, false) 103 } 104 105 if !success { 106 Exit("Warning: errors occurred") 107 } 108 } 109 110 func pointersToFetchForRef(ref string, filter *filepathfilter.Filter) ([]*lfs.WrappedPointer, error) { 111 var pointers []*lfs.WrappedPointer 112 var multiErr error 113 tempgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) { 114 if err != nil { 115 if multiErr != nil { 116 multiErr = fmt.Errorf("%v\n%v", multiErr, err) 117 } else { 118 multiErr = err 119 } 120 return 121 } 122 123 pointers = append(pointers, p) 124 }) 125 126 tempgitscanner.Filter = filter 127 128 if err := tempgitscanner.ScanTree(ref); err != nil { 129 return nil, err 130 } 131 132 tempgitscanner.Close() 133 return pointers, multiErr 134 } 135 136 // Fetch all binaries for a given ref (that we don't have already) 137 func fetchRef(ref string, filter *filepathfilter.Filter) bool { 138 pointers, err := pointersToFetchForRef(ref, filter) 139 if err != nil { 140 Panic(err, "Could not scan for Git LFS files") 141 } 142 return fetchAndReportToChan(pointers, filter, nil) 143 } 144 145 // Fetch all previous versions of objects from since to ref (not including final state at ref) 146 // So this will fetch all the '-' sides of the diff from since to ref 147 func fetchPreviousVersions(ref string, since time.Time, filter *filepathfilter.Filter) bool { 148 var pointers []*lfs.WrappedPointer 149 150 tempgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) { 151 if err != nil { 152 Panic(err, "Could not scan for Git LFS previous versions") 153 return 154 } 155 156 pointers = append(pointers, p) 157 }) 158 159 tempgitscanner.Filter = filter 160 161 if err := tempgitscanner.ScanPreviousVersions(ref, since, nil); err != nil { 162 ExitWithError(err) 163 } 164 165 tempgitscanner.Close() 166 return fetchAndReportToChan(pointers, filter, nil) 167 } 168 169 // Fetch recent objects based on config 170 func fetchRecent(alreadyFetchedRefs []*git.Ref, filter *filepathfilter.Filter) bool { 171 fetchconf := cfg.FetchPruneConfig() 172 173 if fetchconf.FetchRecentRefsDays == 0 && fetchconf.FetchRecentCommitsDays == 0 { 174 return true 175 } 176 177 ok := true 178 // Make a list of what unique commits we've already fetched for to avoid duplicating work 179 uniqueRefShas := make(map[string]string, len(alreadyFetchedRefs)) 180 for _, ref := range alreadyFetchedRefs { 181 uniqueRefShas[ref.Sha] = ref.Name 182 } 183 // First find any other recent refs 184 if fetchconf.FetchRecentRefsDays > 0 { 185 Print("Fetching recent branches within %v days", fetchconf.FetchRecentRefsDays) 186 refsSince := time.Now().AddDate(0, 0, -fetchconf.FetchRecentRefsDays) 187 refs, err := git.RecentBranches(refsSince, fetchconf.FetchRecentRefsIncludeRemotes, cfg.CurrentRemote) 188 if err != nil { 189 Panic(err, "Could not scan for recent refs") 190 } 191 for _, ref := range refs { 192 // Don't fetch for the same SHA twice 193 if prevRefName, ok := uniqueRefShas[ref.Sha]; ok { 194 if ref.Name != prevRefName { 195 tracerx.Printf("Skipping fetch for %v, already fetched via %v", ref.Name, prevRefName) 196 } 197 } else { 198 uniqueRefShas[ref.Sha] = ref.Name 199 Print("Fetching %v", ref.Name) 200 k := fetchRef(ref.Sha, filter) 201 ok = ok && k 202 } 203 } 204 } 205 // For every unique commit we've fetched, check recent commits too 206 if fetchconf.FetchRecentCommitsDays > 0 { 207 for commit, refName := range uniqueRefShas { 208 // We measure from the last commit at the ref 209 summ, err := git.GetCommitSummary(commit) 210 if err != nil { 211 Error("Couldn't scan commits at %v: %v", refName, err) 212 continue 213 } 214 Print("Fetching changes within %v days of %v", fetchconf.FetchRecentCommitsDays, refName) 215 commitsSince := summ.CommitDate.AddDate(0, 0, -fetchconf.FetchRecentCommitsDays) 216 k := fetchPreviousVersions(commit, commitsSince, filter) 217 ok = ok && k 218 } 219 220 } 221 return ok 222 } 223 224 func fetchAll() bool { 225 pointers := scanAll() 226 Print("Fetching objects...") 227 return fetchAndReportToChan(pointers, nil, nil) 228 } 229 230 func scanAll() []*lfs.WrappedPointer { 231 // This could be a long process so use the chan version & report progress 232 Print("Scanning for all objects ever referenced...") 233 spinner := progress.NewSpinner() 234 var numObjs int64 235 236 // use temp gitscanner to collect pointers 237 var pointers []*lfs.WrappedPointer 238 var multiErr error 239 tempgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) { 240 if err != nil { 241 if multiErr != nil { 242 multiErr = fmt.Errorf("%v\n%v", multiErr, err) 243 } else { 244 multiErr = err 245 } 246 return 247 } 248 249 numObjs++ 250 spinner.Print(OutputWriter, fmt.Sprintf("%d objects found", numObjs)) 251 pointers = append(pointers, p) 252 }) 253 254 if err := tempgitscanner.ScanAll(nil); err != nil { 255 Panic(err, "Could not scan for Git LFS files") 256 } 257 258 tempgitscanner.Close() 259 260 if multiErr != nil { 261 Panic(multiErr, "Could not scan for Git LFS files") 262 } 263 264 spinner.Finish(OutputWriter, fmt.Sprintf("%d objects found", numObjs)) 265 return pointers 266 } 267 268 // Fetch and report completion of each OID to a channel (optional, pass nil to skip) 269 // Returns true if all completed with no errors, false if errors were written to stderr/log 270 func fetchAndReportToChan(allpointers []*lfs.WrappedPointer, filter *filepathfilter.Filter, out chan<- *lfs.WrappedPointer) bool { 271 // Lazily initialize the current remote. 272 if len(cfg.CurrentRemote) == 0 { 273 // Actively find the default remote, don't just assume origin 274 defaultRemote, err := git.DefaultRemote() 275 if err != nil { 276 Exit("No default remote") 277 } 278 cfg.CurrentRemote = defaultRemote 279 } 280 281 ready, pointers, meter := readyAndMissingPointers(allpointers, filter) 282 q := newDownloadQueue(getTransferManifest(), cfg.CurrentRemote, tq.WithProgress(meter)) 283 284 if out != nil { 285 // If we already have it, or it won't be fetched 286 // report it to chan immediately to support pull/checkout 287 for _, p := range ready { 288 out <- p 289 } 290 291 dlwatch := q.Watch() 292 293 go func() { 294 // fetch only reports single OID, but OID *might* be referenced by multiple 295 // WrappedPointers if same content is at multiple paths, so map oid->slice 296 oidToPointers := make(map[string][]*lfs.WrappedPointer, len(pointers)) 297 for _, pointer := range pointers { 298 plist := oidToPointers[pointer.Oid] 299 oidToPointers[pointer.Oid] = append(plist, pointer) 300 } 301 302 for oid := range dlwatch { 303 plist, ok := oidToPointers[oid] 304 if !ok { 305 continue 306 } 307 for _, p := range plist { 308 out <- p 309 } 310 } 311 close(out) 312 }() 313 } 314 315 for _, p := range pointers { 316 tracerx.Printf("fetch %v [%v]", p.Name, p.Oid) 317 318 q.Add(downloadTransfer(p)) 319 } 320 321 processQueue := time.Now() 322 q.Wait() 323 tracerx.PerformanceSince("process queue", processQueue) 324 325 ok := true 326 for _, err := range q.Errors() { 327 ok = false 328 FullError(err) 329 } 330 return ok 331 } 332 333 func readyAndMissingPointers(allpointers []*lfs.WrappedPointer, filter *filepathfilter.Filter) ([]*lfs.WrappedPointer, []*lfs.WrappedPointer, *progress.ProgressMeter) { 334 meter := buildProgressMeter(false) 335 seen := make(map[string]bool, len(allpointers)) 336 missing := make([]*lfs.WrappedPointer, 0, len(allpointers)) 337 ready := make([]*lfs.WrappedPointer, 0, len(allpointers)) 338 339 for _, p := range allpointers { 340 // no need to download the same object multiple times 341 if seen[p.Oid] { 342 continue 343 } 344 345 seen[p.Oid] = true 346 347 // no need to download objects that exist locally already 348 lfs.LinkOrCopyFromReference(p.Oid, p.Size) 349 if lfs.ObjectExistsOfSize(p.Oid, p.Size) { 350 ready = append(ready, p) 351 continue 352 } 353 354 missing = append(missing, p) 355 meter.Add(p.Size) 356 } 357 358 return ready, missing, meter 359 } 360 361 func init() { 362 RegisterCommand("fetch", fetchCommand, func(cmd *cobra.Command) { 363 cmd.Flags().StringVarP(&includeArg, "include", "I", "", "Include a list of paths") 364 cmd.Flags().StringVarP(&excludeArg, "exclude", "X", "", "Exclude a list of paths") 365 cmd.Flags().BoolVarP(&fetchRecentArg, "recent", "r", false, "Fetch recent refs & commits") 366 cmd.Flags().BoolVarP(&fetchAllArg, "all", "a", false, "Fetch all LFS files ever referenced") 367 cmd.Flags().BoolVarP(&fetchPruneArg, "prune", "p", false, "After fetching, prune old data") 368 }) 369 }