code.gitea.io/gitea@v1.21.7/cmd/migrate_storage.go (about)

     1  // Copyright 2020 The Gitea Authors. All rights reserved.
     2  // SPDX-License-Identifier: MIT
     3  
     4  package cmd
     5  
     6  import (
     7  	"context"
     8  	"fmt"
     9  	"strings"
    10  
    11  	actions_model "code.gitea.io/gitea/models/actions"
    12  	"code.gitea.io/gitea/models/db"
    13  	git_model "code.gitea.io/gitea/models/git"
    14  	"code.gitea.io/gitea/models/migrations"
    15  	packages_model "code.gitea.io/gitea/models/packages"
    16  	repo_model "code.gitea.io/gitea/models/repo"
    17  	user_model "code.gitea.io/gitea/models/user"
    18  	"code.gitea.io/gitea/modules/log"
    19  	packages_module "code.gitea.io/gitea/modules/packages"
    20  	"code.gitea.io/gitea/modules/setting"
    21  	"code.gitea.io/gitea/modules/storage"
    22  
    23  	"github.com/urfave/cli/v2"
    24  )
    25  
    26  // CmdMigrateStorage represents the available migrate storage sub-command.
    27  var CmdMigrateStorage = &cli.Command{
    28  	Name:        "migrate-storage",
    29  	Usage:       "Migrate the storage",
    30  	Description: "Copies stored files from storage configured in app.ini to parameter-configured storage",
    31  	Action:      runMigrateStorage,
    32  	Flags: []cli.Flag{
    33  		&cli.StringFlag{
    34  			Name:    "type",
    35  			Aliases: []string{"t"},
    36  			Value:   "",
    37  			Usage:   "Type of stored files to copy.  Allowed types: 'attachments', 'lfs', 'avatars', 'repo-avatars', 'repo-archivers', 'packages', 'actions-log'",
    38  		},
    39  		&cli.StringFlag{
    40  			Name:    "storage",
    41  			Aliases: []string{"s"},
    42  			Value:   "",
    43  			Usage:   "New storage type: local (default) or minio",
    44  		},
    45  		&cli.StringFlag{
    46  			Name:    "path",
    47  			Aliases: []string{"p"},
    48  			Value:   "",
    49  			Usage:   "New storage placement if store is local (leave blank for default)",
    50  		},
    51  		&cli.StringFlag{
    52  			Name:  "minio-endpoint",
    53  			Value: "",
    54  			Usage: "Minio storage endpoint",
    55  		},
    56  		&cli.StringFlag{
    57  			Name:  "minio-access-key-id",
    58  			Value: "",
    59  			Usage: "Minio storage accessKeyID",
    60  		},
    61  		&cli.StringFlag{
    62  			Name:  "minio-secret-access-key",
    63  			Value: "",
    64  			Usage: "Minio storage secretAccessKey",
    65  		},
    66  		&cli.StringFlag{
    67  			Name:  "minio-bucket",
    68  			Value: "",
    69  			Usage: "Minio storage bucket",
    70  		},
    71  		&cli.StringFlag{
    72  			Name:  "minio-location",
    73  			Value: "",
    74  			Usage: "Minio storage location to create bucket",
    75  		},
    76  		&cli.StringFlag{
    77  			Name:  "minio-base-path",
    78  			Value: "",
    79  			Usage: "Minio storage base path on the bucket",
    80  		},
    81  		&cli.BoolFlag{
    82  			Name:  "minio-use-ssl",
    83  			Usage: "Enable SSL for minio",
    84  		},
    85  		&cli.BoolFlag{
    86  			Name:  "minio-insecure-skip-verify",
    87  			Usage: "Skip SSL verification",
    88  		},
    89  		&cli.StringFlag{
    90  			Name:  "minio-checksum-algorithm",
    91  			Value: "",
    92  			Usage: "Minio checksum algorithm (default/md5)",
    93  		},
    94  	},
    95  }
    96  
    97  func migrateAttachments(ctx context.Context, dstStorage storage.ObjectStorage) error {
    98  	return db.Iterate(ctx, nil, func(ctx context.Context, attach *repo_model.Attachment) error {
    99  		_, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
   100  		return err
   101  	})
   102  }
   103  
   104  func migrateLFS(ctx context.Context, dstStorage storage.ObjectStorage) error {
   105  	return db.Iterate(ctx, nil, func(ctx context.Context, mo *git_model.LFSMetaObject) error {
   106  		_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
   107  		return err
   108  	})
   109  }
   110  
   111  func migrateAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
   112  	return db.Iterate(ctx, nil, func(ctx context.Context, user *user_model.User) error {
   113  		if user.CustomAvatarRelativePath() == "" {
   114  			return nil
   115  		}
   116  		_, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath())
   117  		return err
   118  	})
   119  }
   120  
   121  func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
   122  	return db.Iterate(ctx, nil, func(ctx context.Context, repo *repo_model.Repository) error {
   123  		if repo.CustomAvatarRelativePath() == "" {
   124  			return nil
   125  		}
   126  		_, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath())
   127  		return err
   128  	})
   129  }
   130  
   131  func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error {
   132  	return db.Iterate(ctx, nil, func(ctx context.Context, archiver *repo_model.RepoArchiver) error {
   133  		p := archiver.RelativePath()
   134  		_, err := storage.Copy(dstStorage, p, storage.RepoArchives, p)
   135  		return err
   136  	})
   137  }
   138  
   139  func migratePackages(ctx context.Context, dstStorage storage.ObjectStorage) error {
   140  	return db.Iterate(ctx, nil, func(ctx context.Context, pb *packages_model.PackageBlob) error {
   141  		p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256))
   142  		_, err := storage.Copy(dstStorage, p, storage.Packages, p)
   143  		return err
   144  	})
   145  }
   146  
   147  func migrateActionsLog(ctx context.Context, dstStorage storage.ObjectStorage) error {
   148  	return db.Iterate(ctx, nil, func(ctx context.Context, task *actions_model.ActionTask) error {
   149  		if task.LogExpired {
   150  			// the log has been cleared
   151  			return nil
   152  		}
   153  		if !task.LogInStorage {
   154  			// running tasks store logs in DBFS
   155  			return nil
   156  		}
   157  		p := task.LogFilename
   158  		_, err := storage.Copy(dstStorage, p, storage.Actions, p)
   159  		return err
   160  	})
   161  }
   162  
   163  func runMigrateStorage(ctx *cli.Context) error {
   164  	stdCtx, cancel := installSignals()
   165  	defer cancel()
   166  
   167  	if err := initDB(stdCtx); err != nil {
   168  		return err
   169  	}
   170  
   171  	log.Info("AppPath: %s", setting.AppPath)
   172  	log.Info("AppWorkPath: %s", setting.AppWorkPath)
   173  	log.Info("Custom path: %s", setting.CustomPath)
   174  	log.Info("Log path: %s", setting.Log.RootPath)
   175  	log.Info("Configuration file: %s", setting.CustomConf)
   176  
   177  	if err := db.InitEngineWithMigration(context.Background(), migrations.Migrate); err != nil {
   178  		log.Fatal("Failed to initialize ORM engine: %v", err)
   179  		return err
   180  	}
   181  
   182  	if err := storage.Init(); err != nil {
   183  		return err
   184  	}
   185  
   186  	var dstStorage storage.ObjectStorage
   187  	var err error
   188  	switch strings.ToLower(ctx.String("storage")) {
   189  	case "":
   190  		fallthrough
   191  	case string(setting.LocalStorageType):
   192  		p := ctx.String("path")
   193  		if p == "" {
   194  			log.Fatal("Path must be given when storage is local")
   195  			return nil
   196  		}
   197  		dstStorage, err = storage.NewLocalStorage(
   198  			stdCtx,
   199  			&setting.Storage{
   200  				Path: p,
   201  			})
   202  	case string(setting.MinioStorageType):
   203  		dstStorage, err = storage.NewMinioStorage(
   204  			stdCtx,
   205  			&setting.Storage{
   206  				MinioConfig: setting.MinioStorageConfig{
   207  					Endpoint:           ctx.String("minio-endpoint"),
   208  					AccessKeyID:        ctx.String("minio-access-key-id"),
   209  					SecretAccessKey:    ctx.String("minio-secret-access-key"),
   210  					Bucket:             ctx.String("minio-bucket"),
   211  					Location:           ctx.String("minio-location"),
   212  					BasePath:           ctx.String("minio-base-path"),
   213  					UseSSL:             ctx.Bool("minio-use-ssl"),
   214  					InsecureSkipVerify: ctx.Bool("minio-insecure-skip-verify"),
   215  					ChecksumAlgorithm:  ctx.String("minio-checksum-algorithm"),
   216  				},
   217  			})
   218  	default:
   219  		return fmt.Errorf("unsupported storage type: %s", ctx.String("storage"))
   220  	}
   221  	if err != nil {
   222  		return err
   223  	}
   224  
   225  	migratedMethods := map[string]func(context.Context, storage.ObjectStorage) error{
   226  		"attachments":    migrateAttachments,
   227  		"lfs":            migrateLFS,
   228  		"avatars":        migrateAvatars,
   229  		"repo-avatars":   migrateRepoAvatars,
   230  		"repo-archivers": migrateRepoArchivers,
   231  		"packages":       migratePackages,
   232  		"actions-log":    migrateActionsLog,
   233  	}
   234  
   235  	tp := strings.ToLower(ctx.String("type"))
   236  	if m, ok := migratedMethods[tp]; ok {
   237  		if err := m(stdCtx, dstStorage); err != nil {
   238  			return err
   239  		}
   240  		log.Info("%s files have successfully been copied to the new storage.", tp)
   241  		return nil
   242  	}
   243  
   244  	return fmt.Errorf("unsupported storage: %s", ctx.String("type"))
   245  }