code.gitea.io/gitea@v1.19.3/modules/lfs/pointer_scanner_nogogit.go (about) 1 // Copyright 2021 The Gitea Authors. All rights reserved. 2 // SPDX-License-Identifier: MIT 3 4 //go:build !gogit 5 6 package lfs 7 8 import ( 9 "bufio" 10 "context" 11 "io" 12 "strconv" 13 "strings" 14 "sync" 15 16 "code.gitea.io/gitea/modules/git" 17 "code.gitea.io/gitea/modules/git/pipeline" 18 ) 19 20 // SearchPointerBlobs scans the whole repository for LFS pointer files 21 func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan chan<- PointerBlob, errChan chan<- error) { 22 basePath := repo.Path 23 24 catFileCheckReader, catFileCheckWriter := io.Pipe() 25 shasToBatchReader, shasToBatchWriter := io.Pipe() 26 catFileBatchReader, catFileBatchWriter := io.Pipe() 27 28 wg := sync.WaitGroup{} 29 wg.Add(4) 30 31 // Create the go-routines in reverse order. 32 33 // 4. Take the output of cat-file --batch and check if each file in turn 34 // to see if they're pointers to files in the LFS store 35 go createPointerResultsFromCatFileBatch(ctx, catFileBatchReader, &wg, pointerChan) 36 37 // 3. Take the shas of the blobs and batch read them 38 go pipeline.CatFileBatch(ctx, shasToBatchReader, catFileBatchWriter, &wg, basePath) 39 40 // 2. From the provided objects restrict to blobs <=1k 41 go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg) 42 43 // 1. Run batch-check on all objects in the repository 44 if git.CheckGitVersionAtLeast("2.6.0") != nil { 45 revListReader, revListWriter := io.Pipe() 46 shasToCheckReader, shasToCheckWriter := io.Pipe() 47 wg.Add(2) 48 go pipeline.CatFileBatchCheck(ctx, shasToCheckReader, catFileCheckWriter, &wg, basePath) 49 go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg) 50 go pipeline.RevListAllObjects(ctx, revListWriter, &wg, basePath, errChan) 51 } else { 52 go pipeline.CatFileBatchCheckAllObjects(ctx, catFileCheckWriter, &wg, basePath, errChan) 53 } 54 wg.Wait() 55 56 close(pointerChan) 57 close(errChan) 58 } 59 60 func createPointerResultsFromCatFileBatch(ctx context.Context, catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- PointerBlob) { 61 defer wg.Done() 62 defer catFileBatchReader.Close() 63 64 bufferedReader := bufio.NewReader(catFileBatchReader) 65 buf := make([]byte, 1025) 66 67 loop: 68 for { 69 select { 70 case <-ctx.Done(): 71 break loop 72 default: 73 } 74 75 // File descriptor line: sha 76 sha, err := bufferedReader.ReadString(' ') 77 if err != nil { 78 _ = catFileBatchReader.CloseWithError(err) 79 break 80 } 81 sha = strings.TrimSpace(sha) 82 // Throw away the blob 83 if _, err := bufferedReader.ReadString(' '); err != nil { 84 _ = catFileBatchReader.CloseWithError(err) 85 break 86 } 87 sizeStr, err := bufferedReader.ReadString('\n') 88 if err != nil { 89 _ = catFileBatchReader.CloseWithError(err) 90 break 91 } 92 size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1]) 93 if err != nil { 94 _ = catFileBatchReader.CloseWithError(err) 95 break 96 } 97 pointerBuf := buf[:size+1] 98 if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil { 99 _ = catFileBatchReader.CloseWithError(err) 100 break 101 } 102 pointerBuf = pointerBuf[:size] 103 // Now we need to check if the pointerBuf is an LFS pointer 104 pointer, _ := ReadPointerFromBuffer(pointerBuf) 105 if !pointer.IsValid() { 106 continue 107 } 108 109 pointerChan <- PointerBlob{Hash: sha, Pointer: pointer} 110 } 111 }