github.com/cozy/cozy-stack@v0.0.0-20240603063001-31110fa4cae1/worker/migrations/migrations.go (about) 1 package migrations 2 3 import ( 4 "context" 5 "errors" 6 "fmt" 7 "os" 8 "runtime" 9 "time" 10 11 "github.com/cozy/cozy-stack/model/instance" 12 "github.com/cozy/cozy-stack/model/job" 13 "github.com/cozy/cozy-stack/model/note" 14 "github.com/cozy/cozy-stack/model/vfs" 15 "github.com/cozy/cozy-stack/model/vfs/vfsswift" 16 "github.com/cozy/cozy-stack/pkg/config/config" 17 "github.com/cozy/cozy-stack/pkg/consts" 18 "github.com/cozy/cozy-stack/pkg/couchdb" 19 "github.com/cozy/cozy-stack/pkg/couchdb/mango" 20 "github.com/cozy/cozy-stack/pkg/i18n" 21 "github.com/cozy/cozy-stack/pkg/logger" 22 "github.com/cozy/cozy-stack/pkg/utils" 23 multierror "github.com/hashicorp/go-multierror" 24 "golang.org/x/sync/errgroup" 25 "golang.org/x/sync/semaphore" 26 27 "github.com/ncw/swift/v2" 28 ) 29 30 const ( 31 swiftV1ToV2 = "swift-v1-to-v2" 32 toSwiftV3 = "to-swift-v3" 33 34 swiftV1ContainerPrefixCozy = "cozy-" 35 swiftV1ContainerPrefixData = "data-" 36 swiftV2ContainerPrefixCozy = "cozy-v2-" 37 swiftV2ContainerPrefixData = "data-v2-" 38 swiftV3ContainerPrefix = "cozy-v3-" 39 40 accountsToOrganization = "accounts-to-organization" 41 notesMimeType = "notes-mime-type" 42 unwantedFolders = "remove-unwanted-folders" 43 ) 44 45 // maxSimultaneousCalls is the maximal number of simultaneous calls to Swift 46 // made by a single migration. 47 const maxSimultaneousCalls = 8 48 49 func init() { 50 job.AddWorker(&job.WorkerConfig{ 51 WorkerType: "migrations", 52 Concurrency: runtime.NumCPU(), 53 MaxExecCount: 1, 54 Reserved: true, 55 WorkerFunc: worker, 56 WorkerCommit: commit, 57 Timeout: 6 * time.Hour, 58 }) 59 } 60 61 type message struct { 62 Type string `json:"type"` 63 } 64 65 func worker(ctx *job.TaskContext) error { 66 var msg message 67 if err := ctx.UnmarshalMessage(&msg); err != nil { 68 return err 69 } 70 71 logger.WithDomain(ctx.Instance.Domain).WithNamespace("migration"). 72 Infof("Start the migration %s", msg.Type) 73 74 switch msg.Type { 75 case toSwiftV3: 76 return migrateToSwiftV3(ctx.Instance.Domain) 77 case swiftV1ToV2: 78 return fmt.Errorf("this migration type is no longer supported") 79 case accountsToOrganization: 80 return migrateAccountsToOrganization(ctx.Instance.Domain) 81 case notesMimeType: 82 return migrateNotesMimeType(ctx.Instance.Domain) 83 case unwantedFolders: 84 return removeUnwantedFolders(ctx.Instance.Domain) 85 default: 86 return fmt.Errorf("unknown migration type %q", msg.Type) 87 } 88 } 89 90 func commit(ctx *job.TaskContext, err error) error { 91 var msg message 92 var migrationType string 93 94 if msgerr := ctx.UnmarshalMessage(&msg); msgerr != nil { 95 migrationType = "" 96 } else { 97 migrationType = msg.Type 98 } 99 100 log := logger.WithDomain(ctx.Instance.Domain).WithNamespace("migration") 101 if err == nil { 102 log.Infof("Migration %s success", migrationType) 103 } else { 104 log.Errorf("Migration %s error: %s", migrationType, err) 105 } 106 return err 107 } 108 109 func pushTrashJob(fs vfs.VFS) func(vfs.TrashJournal) error { 110 return func(journal vfs.TrashJournal) error { 111 return fs.EnsureErased(journal) 112 } 113 } 114 115 func removeUnwantedFolders(domain string) error { 116 inst, err := instance.Get(domain) 117 if err != nil { 118 return err 119 } 120 fs := inst.VFS() 121 122 var errf error 123 removeDir := func(dir *vfs.DirDoc, err error) { 124 if errors.Is(err, os.ErrNotExist) { 125 return 126 } else if err != nil { 127 errf = multierror.Append(errf, err) 128 return 129 } 130 131 hasFiles := false 132 err = vfs.WalkByID(fs, dir.ID(), func(name string, dir *vfs.DirDoc, file *vfs.FileDoc, err error) error { 133 if err != nil { 134 return err 135 } 136 if file != nil { 137 hasFiles = true 138 } 139 return nil 140 }) 141 if err != nil { 142 errf = multierror.Append(errf, err) 143 return 144 } 145 if hasFiles { 146 // This is a guard to avoiding deleting files by mistake. 147 return 148 } 149 push := pushTrashJob(fs) 150 if err = fs.DestroyDirAndContent(dir, push); err != nil { 151 errf = multierror.Append(errf, err) 152 } 153 } 154 155 keepAdministrativeFolder := true 156 keepPhotosFolder := true 157 if ctxSettings, ok := inst.SettingsContext(); ok { 158 if administrativeFolderParam, ok := ctxSettings["init_administrative_folder"]; ok { 159 keepAdministrativeFolder = administrativeFolderParam.(bool) 160 } 161 if photosFolderParam, ok := ctxSettings["init_photos_folder"]; ok { 162 keepPhotosFolder = photosFolderParam.(bool) 163 } 164 } 165 166 if !keepPhotosFolder { 167 name := inst.Translate("Tree Photos") 168 folder, err := fs.DirByPath("/" + name) 169 removeDir(folder, err) 170 } 171 172 if !keepAdministrativeFolder { 173 name := inst.Translate("Tree Administrative") 174 folder, err := fs.DirByPath("/" + name) 175 if err != nil { 176 name = i18n.Translate("Tree Administrative", consts.DefaultLocale, inst.ContextName) 177 folder, err = fs.DirByPath("/" + name) 178 } 179 removeDir(folder, err) 180 181 root, err := fs.DirByID(consts.RootDirID) 182 if err != nil { 183 return err 184 } 185 olddoc := root.Clone().(*vfs.DirDoc) 186 was := len(root.ReferencedBy) 187 root.AddReferencedBy(couchdb.DocReference{ 188 ID: "io.cozy.apps/administrative", 189 Type: consts.Apps, 190 }) 191 if len(root.ReferencedBy) != was { 192 if err := fs.UpdateDirDoc(olddoc, root); err != nil { 193 errf = multierror.Append(errf, err) 194 } 195 } 196 } 197 198 return errf 199 } 200 201 func migrateNotesMimeType(domain string) error { 202 inst, err := instance.Get(domain) 203 if err != nil { 204 return err 205 } 206 log := inst.Logger().WithNamespace("migration") 207 208 var docs []*vfs.FileDoc 209 req := &couchdb.FindRequest{ 210 UseIndex: "by-mime-updated-at", 211 Selector: mango.And( 212 mango.Equal("mime", "text/markdown"), 213 mango.Exists("updated_at"), 214 ), 215 Limit: 1000, 216 } 217 _, err = couchdb.FindDocsRaw(inst, consts.Files, req, &docs) 218 if err != nil { 219 return err 220 } 221 log.Infof("Found %d markdown files", len(docs)) 222 for _, doc := range docs { 223 if _, ok := doc.Metadata["version"]; !ok { 224 log.Infof("Skip file %#v", doc) 225 continue 226 } 227 if err := note.Update(inst, doc.ID()); err != nil { 228 log.Warnf("Cannot change mime-type for note %s: %s", doc.ID(), err) 229 } 230 } 231 232 return nil 233 } 234 235 func migrateToSwiftV3(domain string) error { 236 c := config.GetSwiftConnection() 237 inst, err := instance.Get(domain) 238 if err != nil { 239 return err 240 } 241 log := inst.Logger().WithNamespace("migration") 242 243 var srcContainer, migratedFrom string 244 switch inst.SwiftLayout { 245 case 0: // layout v1 246 srcContainer = swiftV1ContainerPrefixCozy + inst.DBPrefix() 247 migratedFrom = "v1" 248 case 1: // layout v2 249 srcContainer = swiftV2ContainerPrefixCozy + inst.DBPrefix() 250 switch inst.DBPrefix() { 251 case inst.Domain: 252 migratedFrom = "v2a" 253 case inst.Prefix: 254 migratedFrom = "v2b" 255 default: 256 return instance.ErrInvalidSwiftLayout 257 } 258 case 2: // layout v3 259 return nil // Nothing to do! 260 default: 261 return instance.ErrInvalidSwiftLayout 262 } 263 264 log.Infof("Migrating from swift layout %s to swift layout v3", migratedFrom) 265 266 vfs := inst.VFS() 267 root, err := vfs.DirByID(consts.RootDirID) 268 if err != nil { 269 return err 270 } 271 272 mutex := config.Lock().LongOperation(inst, "vfs") 273 if err = mutex.Lock(); err != nil { 274 return err 275 } 276 defer mutex.Unlock() 277 278 ctx := context.Background() 279 dstContainer := swiftV3ContainerPrefix + inst.DBPrefix() 280 if _, _, err = c.Container(ctx, dstContainer); !errors.Is(err, swift.ContainerNotFound) { 281 log.Errorf("Destination container %s already exists or something went wrong. Migration canceled.", dstContainer) 282 return errors.New("Destination container busy") 283 } 284 if err = c.ContainerCreate(ctx, dstContainer, nil); err != nil { 285 return err 286 } 287 defer func() { 288 if err != nil { 289 if err := vfsswift.DeleteContainer(ctx, c, dstContainer); err != nil { 290 log.Errorf("Failed to delete v3 container %s: %s", dstContainer, err) 291 } 292 } 293 }() 294 295 if err = copyTheFilesToSwiftV3(inst, ctx, c, root, srcContainer, dstContainer); err != nil { 296 return err 297 } 298 299 meta := &swift.Metadata{"cozy-migrated-from": migratedFrom} 300 _ = c.ContainerUpdate(ctx, dstContainer, meta.ContainerHeaders()) 301 if in, err := instance.Get(domain); err == nil { 302 inst = in 303 } 304 inst.SwiftLayout = 2 305 if err = instance.Update(inst); err != nil { 306 return err 307 } 308 309 // Migration done. Now clean-up oldies. 310 311 // WARNING: Don't call `err` any error below in this function or the defer func 312 // will delete the new container even if the migration was successful 313 314 if deleteErr := vfs.Delete(); deleteErr != nil { 315 log.Errorf("Failed to delete old %s containers: %s", migratedFrom, deleteErr) 316 } 317 return nil 318 } 319 320 func copyTheFilesToSwiftV3(inst *instance.Instance, ctx context.Context, c *swift.Connection, root *vfs.DirDoc, src, dst string) error { 321 log := logger.WithDomain(inst.Domain). 322 WithNamespace("migration") 323 sem := semaphore.NewWeighted(maxSimultaneousCalls) 324 g, ctx := errgroup.WithContext(context.Background()) 325 fs := inst.VFS() 326 327 var thumbsContainer string 328 switch inst.SwiftLayout { 329 case 0: // layout v1 330 thumbsContainer = swiftV1ContainerPrefixData + inst.Domain 331 case 1: // layout v2 332 thumbsContainer = swiftV2ContainerPrefixData + inst.DBPrefix() 333 default: 334 return instance.ErrInvalidSwiftLayout 335 } 336 337 errw := vfs.WalkAlreadyLocked(fs, root, func(_ string, d *vfs.DirDoc, f *vfs.FileDoc, err error) error { 338 if err != nil { 339 return err 340 } 341 if f == nil { 342 return nil 343 } 344 srcName := getSrcName(inst, f) 345 dstName := getDstName(inst, f) 346 if srcName == "" || dstName == "" { 347 return fmt.Errorf("Unexpected copy: %q -> %q", srcName, dstName) 348 } 349 350 if err := sem.Acquire(ctx, 1); err != nil { 351 return err 352 } 353 g.Go(func() error { 354 defer sem.Release(1) 355 err := utils.RetryWithExpBackoff(3, 200*time.Millisecond, func() error { 356 _, err := c.ObjectCopy(ctx, src, srcName, dst, dstName, nil) 357 return err 358 }) 359 if err != nil { 360 log.Warnf("Cannot copy file from %s %s to %s %s: %s", 361 src, srcName, dst, dstName, err) 362 } 363 return err 364 }) 365 366 // Copy the thumbnails 367 if f.Class == "image" { 368 srcTiny, srcSmall, srcMedium, srcLarge := getThumbsSrcNames(inst, f) 369 dstTiny, dstSmall, dstMedium, dstLarge := getThumbsDstNames(inst, f) 370 if err := sem.Acquire(ctx, 1); err != nil { 371 return err 372 } 373 g.Go(func() error { 374 defer sem.Release(1) 375 _, err := c.ObjectCopy(ctx, thumbsContainer, srcSmall, dst, dstSmall, nil) 376 if err != nil { 377 log.Infof("Cannot copy thumbnail tiny from %s %s to %s %s: %s", 378 thumbsContainer, srcTiny, dst, dstTiny, err) 379 } 380 _, err = c.ObjectCopy(ctx, thumbsContainer, srcSmall, dst, dstSmall, nil) 381 if err != nil { 382 log.Infof("Cannot copy thumbnail small from %s %s to %s %s: %s", 383 thumbsContainer, srcSmall, dst, dstSmall, err) 384 } 385 _, err = c.ObjectCopy(ctx, thumbsContainer, srcMedium, dst, dstMedium, nil) 386 if err != nil { 387 log.Infof("Cannot copy thumbnail medium from %s %s to %s %s: %s", 388 thumbsContainer, srcMedium, dst, dstMedium, err) 389 } 390 _, err = c.ObjectCopy(ctx, thumbsContainer, srcLarge, dst, dstLarge, nil) 391 if err != nil { 392 log.Infof("Cannot copy thumbnail large from %s %s to %s %s: %s", 393 thumbsContainer, srcLarge, dst, dstLarge, err) 394 } 395 return nil 396 }) 397 } 398 return nil 399 }) 400 401 if err := g.Wait(); err != nil { 402 return err 403 } 404 return errw 405 } 406 407 func getSrcName(inst *instance.Instance, f *vfs.FileDoc) string { 408 srcName := "" 409 switch inst.SwiftLayout { 410 case 0: // layout v1 411 srcName = f.DirID + "/" + f.DocName 412 case 1: // layout v2 413 srcName = vfsswift.MakeObjectName(f.DocID) 414 } 415 return srcName 416 } 417 418 // XXX the f FileDoc can be modified to add an InternalID 419 func getDstName(inst *instance.Instance, f *vfs.FileDoc) string { 420 if f.InternalID == "" { 421 old := f.Clone().(*vfs.FileDoc) 422 f.InternalID = vfsswift.NewInternalID() 423 if err := couchdb.UpdateDocWithOld(inst, f, old); err != nil { 424 return "" 425 } 426 } 427 return vfsswift.MakeObjectNameV3(f.DocID, f.InternalID) 428 } 429 430 func getThumbsSrcNames(inst *instance.Instance, f *vfs.FileDoc) (string, string, string, string) { 431 var tiny, small, medium, large string 432 switch inst.SwiftLayout { 433 case 0: // layout v1 434 tiny = fmt.Sprintf("thumbs/%s-tiny", f.DocID) 435 small = fmt.Sprintf("thumbs/%s-small", f.DocID) 436 medium = fmt.Sprintf("thumbs/%s-medium", f.DocID) 437 large = fmt.Sprintf("thumbs/%s-large", f.DocID) 438 case 1: // layout v2 439 obj := vfsswift.MakeObjectName(f.DocID) 440 tiny = fmt.Sprintf("thumbs/%s-tiny", obj) 441 small = fmt.Sprintf("thumbs/%s-small", obj) 442 medium = fmt.Sprintf("thumbs/%s-medium", obj) 443 large = fmt.Sprintf("thumbs/%s-large", obj) 444 } 445 return tiny, small, medium, large 446 } 447 448 func getThumbsDstNames(inst *instance.Instance, f *vfs.FileDoc) (string, string, string, string) { 449 obj := vfsswift.MakeObjectName(f.DocID) 450 tiny := fmt.Sprintf("thumbs/%s-tiny", obj) 451 small := fmt.Sprintf("thumbs/%s-small", obj) 452 medium := fmt.Sprintf("thumbs/%s-medium", obj) 453 large := fmt.Sprintf("thumbs/%s-large", obj) 454 return tiny, small, medium, large 455 }