code.gitea.io/gitea@v1.21.7/services/pull/lfs.go (about)

     1  // Copyright 2019 The Gitea Authors.
     2  // All rights reserved.
     3  // SPDX-License-Identifier: MIT
     4  
     5  package pull
     6  
     7  import (
     8  	"bufio"
     9  	"context"
    10  	"io"
    11  	"strconv"
    12  	"sync"
    13  
    14  	"code.gitea.io/gitea/models/db"
    15  	git_model "code.gitea.io/gitea/models/git"
    16  	issues_model "code.gitea.io/gitea/models/issues"
    17  	"code.gitea.io/gitea/modules/git/pipeline"
    18  	"code.gitea.io/gitea/modules/lfs"
    19  	"code.gitea.io/gitea/modules/log"
    20  )
    21  
    22  // LFSPush pushes lfs objects referred to in new commits in the head repository from the base repository
    23  func LFSPush(ctx context.Context, tmpBasePath, mergeHeadSHA, mergeBaseSHA string, pr *issues_model.PullRequest) error {
    24  	// Now we have to implement git lfs push
    25  	// git rev-list --objects --filter=blob:limit=1k HEAD --not base
    26  	// pass blob shas in to git cat-file --batch-check (possibly unnecessary)
    27  	// ensure only blobs and <=1k size then pass in to git cat-file --batch
    28  	// to read each sha and check each as a pointer
    29  	// Then if they are lfs -> add them to the baseRepo
    30  	revListReader, revListWriter := io.Pipe()
    31  	shasToCheckReader, shasToCheckWriter := io.Pipe()
    32  	catFileCheckReader, catFileCheckWriter := io.Pipe()
    33  	shasToBatchReader, shasToBatchWriter := io.Pipe()
    34  	catFileBatchReader, catFileBatchWriter := io.Pipe()
    35  	errChan := make(chan error, 1)
    36  	wg := sync.WaitGroup{}
    37  	wg.Add(6)
    38  	// Create the go-routines in reverse order.
    39  
    40  	// 6. Take the output of cat-file --batch and check if each file in turn
    41  	// to see if they're pointers to files in the LFS store associated with
    42  	// the head repo and add them to the base repo if so
    43  	go createLFSMetaObjectsFromCatFileBatch(catFileBatchReader, &wg, pr)
    44  
    45  	// 5. Take the shas of the blobs and batch read them
    46  	go pipeline.CatFileBatch(ctx, shasToBatchReader, catFileBatchWriter, &wg, tmpBasePath)
    47  
    48  	// 4. From the provided objects restrict to blobs <=1k
    49  	go pipeline.BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader, shasToBatchWriter, &wg)
    50  
    51  	// 3. Run batch-check on the objects retrieved from rev-list
    52  	go pipeline.CatFileBatchCheck(ctx, shasToCheckReader, catFileCheckWriter, &wg, tmpBasePath)
    53  
    54  	// 2. Check each object retrieved rejecting those without names as they will be commits or trees
    55  	go pipeline.BlobsFromRevListObjects(revListReader, shasToCheckWriter, &wg)
    56  
    57  	// 1. Run rev-list objects from mergeHead to mergeBase
    58  	go pipeline.RevListObjects(ctx, revListWriter, &wg, tmpBasePath, mergeHeadSHA, mergeBaseSHA, errChan)
    59  
    60  	wg.Wait()
    61  	select {
    62  	case err, has := <-errChan:
    63  		if has {
    64  			return err
    65  		}
    66  	default:
    67  	}
    68  	return nil
    69  }
    70  
    71  func createLFSMetaObjectsFromCatFileBatch(catFileBatchReader *io.PipeReader, wg *sync.WaitGroup, pr *issues_model.PullRequest) {
    72  	defer wg.Done()
    73  	defer catFileBatchReader.Close()
    74  
    75  	contentStore := lfs.NewContentStore()
    76  
    77  	bufferedReader := bufio.NewReader(catFileBatchReader)
    78  	buf := make([]byte, 1025)
    79  	for {
    80  		// File descriptor line: sha
    81  		_, err := bufferedReader.ReadString(' ')
    82  		if err != nil {
    83  			_ = catFileBatchReader.CloseWithError(err)
    84  			break
    85  		}
    86  		// Throw away the blob
    87  		if _, err := bufferedReader.ReadString(' '); err != nil {
    88  			_ = catFileBatchReader.CloseWithError(err)
    89  			break
    90  		}
    91  		sizeStr, err := bufferedReader.ReadString('\n')
    92  		if err != nil {
    93  			_ = catFileBatchReader.CloseWithError(err)
    94  			break
    95  		}
    96  		size, err := strconv.Atoi(sizeStr[:len(sizeStr)-1])
    97  		if err != nil {
    98  			_ = catFileBatchReader.CloseWithError(err)
    99  			break
   100  		}
   101  		pointerBuf := buf[:size+1]
   102  		if _, err := io.ReadFull(bufferedReader, pointerBuf); err != nil {
   103  			_ = catFileBatchReader.CloseWithError(err)
   104  			break
   105  		}
   106  		pointerBuf = pointerBuf[:size]
   107  		// Now we need to check if the pointerBuf is an LFS pointer
   108  		pointer, _ := lfs.ReadPointerFromBuffer(pointerBuf)
   109  		if !pointer.IsValid() {
   110  			continue
   111  		}
   112  
   113  		exist, _ := contentStore.Exists(pointer)
   114  		if !exist {
   115  			continue
   116  		}
   117  
   118  		// Then we need to check that this pointer is in the db
   119  		if _, err := git_model.GetLFSMetaObjectByOid(db.DefaultContext, pr.HeadRepoID, pointer.Oid); err != nil {
   120  			if err == git_model.ErrLFSObjectNotExist {
   121  				log.Warn("During merge of: %d in %-v, there is a pointer to LFS Oid: %s which although present in the LFS store is not associated with the head repo %-v", pr.Index, pr.BaseRepo, pointer.Oid, pr.HeadRepo)
   122  				continue
   123  			}
   124  			_ = catFileBatchReader.CloseWithError(err)
   125  			break
   126  		}
   127  		// OK we have a pointer that is associated with the head repo
   128  		// and is actually a file in the LFS
   129  		// Therefore it should be associated with the base repo
   130  		meta := &git_model.LFSMetaObject{Pointer: pointer}
   131  		meta.RepositoryID = pr.BaseRepoID
   132  		if _, err := git_model.NewLFSMetaObject(db.DefaultContext, meta); err != nil {
   133  			_ = catFileBatchReader.CloseWithError(err)
   134  			break
   135  		}
   136  	}
   137  }