code.gitea.io/gitea@v1.21.7/cmd/migrate_storage.go (about) 1 // Copyright 2020 The Gitea Authors. All rights reserved. 2 // SPDX-License-Identifier: MIT 3 4 package cmd 5 6 import ( 7 "context" 8 "fmt" 9 "strings" 10 11 actions_model "code.gitea.io/gitea/models/actions" 12 "code.gitea.io/gitea/models/db" 13 git_model "code.gitea.io/gitea/models/git" 14 "code.gitea.io/gitea/models/migrations" 15 packages_model "code.gitea.io/gitea/models/packages" 16 repo_model "code.gitea.io/gitea/models/repo" 17 user_model "code.gitea.io/gitea/models/user" 18 "code.gitea.io/gitea/modules/log" 19 packages_module "code.gitea.io/gitea/modules/packages" 20 "code.gitea.io/gitea/modules/setting" 21 "code.gitea.io/gitea/modules/storage" 22 23 "github.com/urfave/cli/v2" 24 ) 25 26 // CmdMigrateStorage represents the available migrate storage sub-command. 27 var CmdMigrateStorage = &cli.Command{ 28 Name: "migrate-storage", 29 Usage: "Migrate the storage", 30 Description: "Copies stored files from storage configured in app.ini to parameter-configured storage", 31 Action: runMigrateStorage, 32 Flags: []cli.Flag{ 33 &cli.StringFlag{ 34 Name: "type", 35 Aliases: []string{"t"}, 36 Value: "", 37 Usage: "Type of stored files to copy. Allowed types: 'attachments', 'lfs', 'avatars', 'repo-avatars', 'repo-archivers', 'packages', 'actions-log'", 38 }, 39 &cli.StringFlag{ 40 Name: "storage", 41 Aliases: []string{"s"}, 42 Value: "", 43 Usage: "New storage type: local (default) or minio", 44 }, 45 &cli.StringFlag{ 46 Name: "path", 47 Aliases: []string{"p"}, 48 Value: "", 49 Usage: "New storage placement if store is local (leave blank for default)", 50 }, 51 &cli.StringFlag{ 52 Name: "minio-endpoint", 53 Value: "", 54 Usage: "Minio storage endpoint", 55 }, 56 &cli.StringFlag{ 57 Name: "minio-access-key-id", 58 Value: "", 59 Usage: "Minio storage accessKeyID", 60 }, 61 &cli.StringFlag{ 62 Name: "minio-secret-access-key", 63 Value: "", 64 Usage: "Minio storage secretAccessKey", 65 }, 66 &cli.StringFlag{ 67 Name: "minio-bucket", 68 Value: "", 69 Usage: "Minio storage bucket", 70 }, 71 &cli.StringFlag{ 72 Name: "minio-location", 73 Value: "", 74 Usage: "Minio storage location to create bucket", 75 }, 76 &cli.StringFlag{ 77 Name: "minio-base-path", 78 Value: "", 79 Usage: "Minio storage base path on the bucket", 80 }, 81 &cli.BoolFlag{ 82 Name: "minio-use-ssl", 83 Usage: "Enable SSL for minio", 84 }, 85 &cli.BoolFlag{ 86 Name: "minio-insecure-skip-verify", 87 Usage: "Skip SSL verification", 88 }, 89 &cli.StringFlag{ 90 Name: "minio-checksum-algorithm", 91 Value: "", 92 Usage: "Minio checksum algorithm (default/md5)", 93 }, 94 }, 95 } 96 97 func migrateAttachments(ctx context.Context, dstStorage storage.ObjectStorage) error { 98 return db.Iterate(ctx, nil, func(ctx context.Context, attach *repo_model.Attachment) error { 99 _, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath()) 100 return err 101 }) 102 } 103 104 func migrateLFS(ctx context.Context, dstStorage storage.ObjectStorage) error { 105 return db.Iterate(ctx, nil, func(ctx context.Context, mo *git_model.LFSMetaObject) error { 106 _, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath()) 107 return err 108 }) 109 } 110 111 func migrateAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error { 112 return db.Iterate(ctx, nil, func(ctx context.Context, user *user_model.User) error { 113 if user.CustomAvatarRelativePath() == "" { 114 return nil 115 } 116 _, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath()) 117 return err 118 }) 119 } 120 121 func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error { 122 return db.Iterate(ctx, nil, func(ctx context.Context, repo *repo_model.Repository) error { 123 if repo.CustomAvatarRelativePath() == "" { 124 return nil 125 } 126 _, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath()) 127 return err 128 }) 129 } 130 131 func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error { 132 return db.Iterate(ctx, nil, func(ctx context.Context, archiver *repo_model.RepoArchiver) error { 133 p := archiver.RelativePath() 134 _, err := storage.Copy(dstStorage, p, storage.RepoArchives, p) 135 return err 136 }) 137 } 138 139 func migratePackages(ctx context.Context, dstStorage storage.ObjectStorage) error { 140 return db.Iterate(ctx, nil, func(ctx context.Context, pb *packages_model.PackageBlob) error { 141 p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256)) 142 _, err := storage.Copy(dstStorage, p, storage.Packages, p) 143 return err 144 }) 145 } 146 147 func migrateActionsLog(ctx context.Context, dstStorage storage.ObjectStorage) error { 148 return db.Iterate(ctx, nil, func(ctx context.Context, task *actions_model.ActionTask) error { 149 if task.LogExpired { 150 // the log has been cleared 151 return nil 152 } 153 if !task.LogInStorage { 154 // running tasks store logs in DBFS 155 return nil 156 } 157 p := task.LogFilename 158 _, err := storage.Copy(dstStorage, p, storage.Actions, p) 159 return err 160 }) 161 } 162 163 func runMigrateStorage(ctx *cli.Context) error { 164 stdCtx, cancel := installSignals() 165 defer cancel() 166 167 if err := initDB(stdCtx); err != nil { 168 return err 169 } 170 171 log.Info("AppPath: %s", setting.AppPath) 172 log.Info("AppWorkPath: %s", setting.AppWorkPath) 173 log.Info("Custom path: %s", setting.CustomPath) 174 log.Info("Log path: %s", setting.Log.RootPath) 175 log.Info("Configuration file: %s", setting.CustomConf) 176 177 if err := db.InitEngineWithMigration(context.Background(), migrations.Migrate); err != nil { 178 log.Fatal("Failed to initialize ORM engine: %v", err) 179 return err 180 } 181 182 if err := storage.Init(); err != nil { 183 return err 184 } 185 186 var dstStorage storage.ObjectStorage 187 var err error 188 switch strings.ToLower(ctx.String("storage")) { 189 case "": 190 fallthrough 191 case string(setting.LocalStorageType): 192 p := ctx.String("path") 193 if p == "" { 194 log.Fatal("Path must be given when storage is local") 195 return nil 196 } 197 dstStorage, err = storage.NewLocalStorage( 198 stdCtx, 199 &setting.Storage{ 200 Path: p, 201 }) 202 case string(setting.MinioStorageType): 203 dstStorage, err = storage.NewMinioStorage( 204 stdCtx, 205 &setting.Storage{ 206 MinioConfig: setting.MinioStorageConfig{ 207 Endpoint: ctx.String("minio-endpoint"), 208 AccessKeyID: ctx.String("minio-access-key-id"), 209 SecretAccessKey: ctx.String("minio-secret-access-key"), 210 Bucket: ctx.String("minio-bucket"), 211 Location: ctx.String("minio-location"), 212 BasePath: ctx.String("minio-base-path"), 213 UseSSL: ctx.Bool("minio-use-ssl"), 214 InsecureSkipVerify: ctx.Bool("minio-insecure-skip-verify"), 215 ChecksumAlgorithm: ctx.String("minio-checksum-algorithm"), 216 }, 217 }) 218 default: 219 return fmt.Errorf("unsupported storage type: %s", ctx.String("storage")) 220 } 221 if err != nil { 222 return err 223 } 224 225 migratedMethods := map[string]func(context.Context, storage.ObjectStorage) error{ 226 "attachments": migrateAttachments, 227 "lfs": migrateLFS, 228 "avatars": migrateAvatars, 229 "repo-avatars": migrateRepoAvatars, 230 "repo-archivers": migrateRepoArchivers, 231 "packages": migratePackages, 232 "actions-log": migrateActionsLog, 233 } 234 235 tp := strings.ToLower(ctx.String("type")) 236 if m, ok := migratedMethods[tp]; ok { 237 if err := m(stdCtx, dstStorage); err != nil { 238 return err 239 } 240 log.Info("%s files have successfully been copied to the new storage.", tp) 241 return nil 242 } 243 244 return fmt.Errorf("unsupported storage: %s", ctx.String("type")) 245 }