code.gitea.io/gitea@v1.22.3/services/repository/lfs.go (about)

     1  // Copyright 2022 The Gitea Authors. All rights reserved.
     2  // SPDX-License-Identifier: MIT
     3  
     4  package repository
     5  
     6  import (
     7  	"context"
     8  	"errors"
     9  	"fmt"
    10  	"time"
    11  
    12  	git_model "code.gitea.io/gitea/models/git"
    13  	repo_model "code.gitea.io/gitea/models/repo"
    14  	"code.gitea.io/gitea/modules/git"
    15  	"code.gitea.io/gitea/modules/gitrepo"
    16  	"code.gitea.io/gitea/modules/lfs"
    17  	"code.gitea.io/gitea/modules/log"
    18  	"code.gitea.io/gitea/modules/setting"
    19  	"code.gitea.io/gitea/modules/timeutil"
    20  )
    21  
    22  // GarbageCollectLFSMetaObjectsOptions provides options for GarbageCollectLFSMetaObjects function
    23  type GarbageCollectLFSMetaObjectsOptions struct {
    24  	LogDetail                func(format string, v ...any)
    25  	AutoFix                  bool
    26  	OlderThan                time.Time
    27  	UpdatedLessRecentlyThan  time.Time
    28  	NumberToCheckPerRepo     int64
    29  	ProportionToCheckPerRepo float64
    30  }
    31  
    32  // GarbageCollectLFSMetaObjects garbage collects LFS objects for all repositories
    33  func GarbageCollectLFSMetaObjects(ctx context.Context, opts GarbageCollectLFSMetaObjectsOptions) error {
    34  	log.Trace("Doing: GarbageCollectLFSMetaObjects")
    35  	defer log.Trace("Finished: GarbageCollectLFSMetaObjects")
    36  
    37  	if opts.LogDetail == nil {
    38  		opts.LogDetail = log.Debug
    39  	}
    40  
    41  	if !setting.LFS.StartServer {
    42  		opts.LogDetail("LFS support is disabled")
    43  		return nil
    44  	}
    45  
    46  	return git_model.IterateRepositoryIDsWithLFSMetaObjects(ctx, func(ctx context.Context, repoID, count int64) error {
    47  		repo, err := repo_model.GetRepositoryByID(ctx, repoID)
    48  		if err != nil {
    49  			return err
    50  		}
    51  
    52  		if newMinimum := int64(float64(count) * opts.ProportionToCheckPerRepo); newMinimum > opts.NumberToCheckPerRepo && opts.NumberToCheckPerRepo != 0 {
    53  			opts.NumberToCheckPerRepo = newMinimum
    54  		}
    55  		return GarbageCollectLFSMetaObjectsForRepo(ctx, repo, opts)
    56  	})
    57  }
    58  
    59  // GarbageCollectLFSMetaObjectsForRepo garbage collects LFS objects for a specific repository
    60  func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.Repository, opts GarbageCollectLFSMetaObjectsOptions) error {
    61  	opts.LogDetail("Checking %-v", repo)
    62  	total, orphaned, collected, deleted := int64(0), 0, 0, 0
    63  	defer func() {
    64  		if orphaned == 0 {
    65  			opts.LogDetail("Found %d total LFSMetaObjects in %-v", total, repo)
    66  		} else if !opts.AutoFix {
    67  			opts.LogDetail("Found %d/%d orphaned LFSMetaObjects in %-v", orphaned, total, repo)
    68  		} else {
    69  			opts.LogDetail("Collected %d/%d orphaned/%d total LFSMetaObjects in %-v. %d removed from storage.", collected, orphaned, total, repo, deleted)
    70  		}
    71  	}()
    72  
    73  	gitRepo, err := gitrepo.OpenRepository(ctx, repo)
    74  	if err != nil {
    75  		log.Error("Unable to open git repository %-v: %v", repo, err)
    76  		return err
    77  	}
    78  	defer gitRepo.Close()
    79  
    80  	store := lfs.NewContentStore()
    81  	errStop := errors.New("STOPERR")
    82  	objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName)
    83  
    84  	err = git_model.IterateLFSMetaObjectsForRepo(ctx, repo.ID, func(ctx context.Context, metaObject *git_model.LFSMetaObject, count int64) error {
    85  		if opts.NumberToCheckPerRepo > 0 && total > opts.NumberToCheckPerRepo {
    86  			return errStop
    87  		}
    88  		total++
    89  		pointerSha := git.ComputeBlobHash(objectFormat, []byte(metaObject.Pointer.StringContent()))
    90  
    91  		if gitRepo.IsObjectExist(pointerSha.String()) {
    92  			return git_model.MarkLFSMetaObject(ctx, metaObject.ID)
    93  		}
    94  		orphaned++
    95  
    96  		if !opts.AutoFix {
    97  			return nil
    98  		}
    99  		// Non-existent pointer file
   100  		_, err = git_model.RemoveLFSMetaObjectByOidFn(ctx, repo.ID, metaObject.Oid, func(count int64) error {
   101  			if count > 0 {
   102  				return nil
   103  			}
   104  
   105  			if err := store.Delete(metaObject.RelativePath()); err != nil {
   106  				log.Error("Unable to remove lfs metaobject %s from store: %v", metaObject.Oid, err)
   107  			}
   108  			deleted++
   109  			return nil
   110  		})
   111  		if err != nil {
   112  			return fmt.Errorf("unable to remove meta-object %s in %s: %w", metaObject.Oid, repo.FullName(), err)
   113  		}
   114  		collected++
   115  
   116  		return nil
   117  	}, &git_model.IterateLFSMetaObjectsForRepoOptions{
   118  		// Only attempt to garbage collect lfs meta objects older than a week as the order of git lfs upload
   119  		// and git object upload is not necessarily guaranteed. It's possible to imagine a situation whereby
   120  		// an LFS object is uploaded but the git branch is not uploaded immediately, or there are some rapid
   121  		// changes in new branches that might lead to lfs objects becoming temporarily unassociated with git
   122  		// objects.
   123  		//
   124  		// It is likely that a week is potentially excessive but it should definitely be enough that any
   125  		// unassociated LFS object is genuinely unassociated.
   126  		OlderThan:                 timeutil.TimeStamp(opts.OlderThan.Unix()),
   127  		UpdatedLessRecentlyThan:   timeutil.TimeStamp(opts.UpdatedLessRecentlyThan.Unix()),
   128  		OrderByUpdated:            true,
   129  		LoopFunctionAlwaysUpdates: true,
   130  	})
   131  
   132  	if err == errStop {
   133  		opts.LogDetail("Processing stopped at %d total LFSMetaObjects in %-v", total, repo)
   134  		return nil
   135  	} else if err != nil {
   136  		return err
   137  	}
   138  	return nil
   139  }