code.gitea.io/gitea@v1.19.3/modules/lfs/pointer_scanner_nogogit.go (about)

     1  // Copyright 2021 The Gitea Authors. All rights reserved.
     2  // SPDX-License-Identifier: MIT
     3  
     4  //go:build !gogit
     5  
     6  package lfs
     7  
     8  import (
     9  	"bufio"
    10  	"context"
    11  	"io"
    12  	"strconv"
    13  	"strings"
    14  	"sync"
    15  
    16  	"code.gitea.io/gitea/modules/git"
    17  	"code.gitea.io/gitea/modules/git/pipeline"
    18  )
    19  
    20  // SearchPointerBlobs scans the whole repository for LFS pointer files
    21  func SearchPointerBlobs(ctx context.Context, repo *git.Repository, pointerChan chan<- PointerBlob, errChan chan<- error) {
    22  	basePath := repo.Path
    23  
    24  	catFileCheckReader, catFileCheckWriter := io.Pipe()
    25  	shasToBatchReader, shasToBatchWriter := io.Pipe()
    26  	catFileBatchReader, catFileBatchWriter := io.Pipe()
    27  
    28  	wg := sync.WaitGroup{}
    29  	wg.Add(4)
    30  
    31  	// Create the go-routines in reverse order.
    32  
    33  	// 4. Take the output of cat-file --batch and check if each file in turn
    34  	// to see if they're pointers to files in the LFS store
    35  	go createPointerResultsFromCatFileBatch(ctx, catFileBatchReader, &wg, pointerChan)
    36  
    37  	// 3. Take the shas of the blobs and batch read them
    38  	go pipeline.CatFileBatch(ctx, shasToBatchReader, catFileBatchWriter, &wg, basePath)
    39  
    40  	// 2. From the provided objects restrict to blobs <=1k
    41  	go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg)
    42  
    43  	// 1. Run batch-check on all objects in the repository
    44  	if git.CheckGitVersionAtLeast("2.6.0") != nil {
    45  		revListReader, revListWriter := io.Pipe()
    46  		shasToCheckReader, shasToCheckWriter := io.Pipe()
    47  		wg.Add(2)
    48  		go pipeline.CatFileBatchCheck(ctx, shasToCheckReader, catFileCheckWriter, &wg, basePath)
    49  		go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg)
    50  		go pipeline.RevListAllObjects(ctx, revListWriter, &wg, basePath, errChan)
    51  	} else {
    52  		go pipeline.CatFileBatchCheckAllObjects(ctx, catFileCheckWriter, &wg, basePath, errChan)
    53  	}
    54  	wg.Wait()
    55  
    56  	close(pointerChan)
    57  	close(errChan)
    58  }
    59  
    60  func createPointerResultsFromCatFileBatch(ctx context.Context, catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pointerChan chan<- PointerBlob) {
    61  	defer wg.Done()
    62  	defer catFileBatchReader.Close()
    63  
    64  	bufferedReader := bufio.NewReader(catFileBatchReader)
    65  	buf := make([]byte, 1025)
    66  
    67  loop:
    68  	for {
    69  		select {
    70  		case <-ctx.Done():
    71  			break loop
    72  		default:
    73  		}
    74  
    75  		// File descriptor line: sha
    76  		sha, err := bufferedReader.ReadString(' ')
    77  		if err != nil {
    78  			_ = catFileBatchReader.CloseWithError(err)
    79  			break
    80  		}
    81  		sha = strings.TrimSpace(sha)
    82  		// Throw away the blob
    83  		if _, err := bufferedReader.ReadString(' '); err != nil {
    84  			_ = catFileBatchReader.CloseWithError(err)
    85  			break
    86  		}
    87  		sizeStr, err := bufferedReader.ReadString('\n')
    88  		if err != nil {
    89  			_ = catFileBatchReader.CloseWithError(err)
    90  			break
    91  		}
    92  		size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1])
    93  		if err != nil {
    94  			_ = catFileBatchReader.CloseWithError(err)
    95  			break
    96  		}
    97  		pointerBuf := buf[:size+1]
    98  		if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil {
    99  			_ = catFileBatchReader.CloseWithError(err)
   100  			break
   101  		}
   102  		pointerBuf = pointerBuf[:size]
   103  		// Now we need to check if the pointerBuf is an LFS pointer
   104  		pointer, _ := ReadPointerFromBuffer(pointerBuf)
   105  		if !pointer.IsValid() {
   106  			continue
   107  		}
   108  
   109  		pointerChan <- PointerBlob{Hash: sha, Pointer: pointer}
   110  	}
   111  }