github.com/cozy/cozy-stack@v0.0.0-20240603063001-31110fa4cae1/worker/migrations/migrations.go (about)

     1  package migrations
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"os"
     8  	"runtime"
     9  	"time"
    10  
    11  	"github.com/cozy/cozy-stack/model/instance"
    12  	"github.com/cozy/cozy-stack/model/job"
    13  	"github.com/cozy/cozy-stack/model/note"
    14  	"github.com/cozy/cozy-stack/model/vfs"
    15  	"github.com/cozy/cozy-stack/model/vfs/vfsswift"
    16  	"github.com/cozy/cozy-stack/pkg/config/config"
    17  	"github.com/cozy/cozy-stack/pkg/consts"
    18  	"github.com/cozy/cozy-stack/pkg/couchdb"
    19  	"github.com/cozy/cozy-stack/pkg/couchdb/mango"
    20  	"github.com/cozy/cozy-stack/pkg/i18n"
    21  	"github.com/cozy/cozy-stack/pkg/logger"
    22  	"github.com/cozy/cozy-stack/pkg/utils"
    23  	multierror "github.com/hashicorp/go-multierror"
    24  	"golang.org/x/sync/errgroup"
    25  	"golang.org/x/sync/semaphore"
    26  
    27  	"github.com/ncw/swift/v2"
    28  )
    29  
    30  const (
    31  	swiftV1ToV2 = "swift-v1-to-v2"
    32  	toSwiftV3   = "to-swift-v3"
    33  
    34  	swiftV1ContainerPrefixCozy = "cozy-"
    35  	swiftV1ContainerPrefixData = "data-"
    36  	swiftV2ContainerPrefixCozy = "cozy-v2-"
    37  	swiftV2ContainerPrefixData = "data-v2-"
    38  	swiftV3ContainerPrefix     = "cozy-v3-"
    39  
    40  	accountsToOrganization = "accounts-to-organization"
    41  	notesMimeType          = "notes-mime-type"
    42  	unwantedFolders        = "remove-unwanted-folders"
    43  )
    44  
    45  // maxSimultaneousCalls is the maximal number of simultaneous calls to Swift
    46  // made by a single migration.
    47  const maxSimultaneousCalls = 8
    48  
    49  func init() {
    50  	job.AddWorker(&job.WorkerConfig{
    51  		WorkerType:   "migrations",
    52  		Concurrency:  runtime.NumCPU(),
    53  		MaxExecCount: 1,
    54  		Reserved:     true,
    55  		WorkerFunc:   worker,
    56  		WorkerCommit: commit,
    57  		Timeout:      6 * time.Hour,
    58  	})
    59  }
    60  
    61  type message struct {
    62  	Type string `json:"type"`
    63  }
    64  
    65  func worker(ctx *job.TaskContext) error {
    66  	var msg message
    67  	if err := ctx.UnmarshalMessage(&msg); err != nil {
    68  		return err
    69  	}
    70  
    71  	logger.WithDomain(ctx.Instance.Domain).WithNamespace("migration").
    72  		Infof("Start the migration %s", msg.Type)
    73  
    74  	switch msg.Type {
    75  	case toSwiftV3:
    76  		return migrateToSwiftV3(ctx.Instance.Domain)
    77  	case swiftV1ToV2:
    78  		return fmt.Errorf("this migration type is no longer supported")
    79  	case accountsToOrganization:
    80  		return migrateAccountsToOrganization(ctx.Instance.Domain)
    81  	case notesMimeType:
    82  		return migrateNotesMimeType(ctx.Instance.Domain)
    83  	case unwantedFolders:
    84  		return removeUnwantedFolders(ctx.Instance.Domain)
    85  	default:
    86  		return fmt.Errorf("unknown migration type %q", msg.Type)
    87  	}
    88  }
    89  
    90  func commit(ctx *job.TaskContext, err error) error {
    91  	var msg message
    92  	var migrationType string
    93  
    94  	if msgerr := ctx.UnmarshalMessage(&msg); msgerr != nil {
    95  		migrationType = ""
    96  	} else {
    97  		migrationType = msg.Type
    98  	}
    99  
   100  	log := logger.WithDomain(ctx.Instance.Domain).WithNamespace("migration")
   101  	if err == nil {
   102  		log.Infof("Migration %s success", migrationType)
   103  	} else {
   104  		log.Errorf("Migration %s error: %s", migrationType, err)
   105  	}
   106  	return err
   107  }
   108  
   109  func pushTrashJob(fs vfs.VFS) func(vfs.TrashJournal) error {
   110  	return func(journal vfs.TrashJournal) error {
   111  		return fs.EnsureErased(journal)
   112  	}
   113  }
   114  
   115  func removeUnwantedFolders(domain string) error {
   116  	inst, err := instance.Get(domain)
   117  	if err != nil {
   118  		return err
   119  	}
   120  	fs := inst.VFS()
   121  
   122  	var errf error
   123  	removeDir := func(dir *vfs.DirDoc, err error) {
   124  		if errors.Is(err, os.ErrNotExist) {
   125  			return
   126  		} else if err != nil {
   127  			errf = multierror.Append(errf, err)
   128  			return
   129  		}
   130  
   131  		hasFiles := false
   132  		err = vfs.WalkByID(fs, dir.ID(), func(name string, dir *vfs.DirDoc, file *vfs.FileDoc, err error) error {
   133  			if err != nil {
   134  				return err
   135  			}
   136  			if file != nil {
   137  				hasFiles = true
   138  			}
   139  			return nil
   140  		})
   141  		if err != nil {
   142  			errf = multierror.Append(errf, err)
   143  			return
   144  		}
   145  		if hasFiles {
   146  			// This is a guard to avoiding deleting files by mistake.
   147  			return
   148  		}
   149  		push := pushTrashJob(fs)
   150  		if err = fs.DestroyDirAndContent(dir, push); err != nil {
   151  			errf = multierror.Append(errf, err)
   152  		}
   153  	}
   154  
   155  	keepAdministrativeFolder := true
   156  	keepPhotosFolder := true
   157  	if ctxSettings, ok := inst.SettingsContext(); ok {
   158  		if administrativeFolderParam, ok := ctxSettings["init_administrative_folder"]; ok {
   159  			keepAdministrativeFolder = administrativeFolderParam.(bool)
   160  		}
   161  		if photosFolderParam, ok := ctxSettings["init_photos_folder"]; ok {
   162  			keepPhotosFolder = photosFolderParam.(bool)
   163  		}
   164  	}
   165  
   166  	if !keepPhotosFolder {
   167  		name := inst.Translate("Tree Photos")
   168  		folder, err := fs.DirByPath("/" + name)
   169  		removeDir(folder, err)
   170  	}
   171  
   172  	if !keepAdministrativeFolder {
   173  		name := inst.Translate("Tree Administrative")
   174  		folder, err := fs.DirByPath("/" + name)
   175  		if err != nil {
   176  			name = i18n.Translate("Tree Administrative", consts.DefaultLocale, inst.ContextName)
   177  			folder, err = fs.DirByPath("/" + name)
   178  		}
   179  		removeDir(folder, err)
   180  
   181  		root, err := fs.DirByID(consts.RootDirID)
   182  		if err != nil {
   183  			return err
   184  		}
   185  		olddoc := root.Clone().(*vfs.DirDoc)
   186  		was := len(root.ReferencedBy)
   187  		root.AddReferencedBy(couchdb.DocReference{
   188  			ID:   "io.cozy.apps/administrative",
   189  			Type: consts.Apps,
   190  		})
   191  		if len(root.ReferencedBy) != was {
   192  			if err := fs.UpdateDirDoc(olddoc, root); err != nil {
   193  				errf = multierror.Append(errf, err)
   194  			}
   195  		}
   196  	}
   197  
   198  	return errf
   199  }
   200  
   201  func migrateNotesMimeType(domain string) error {
   202  	inst, err := instance.Get(domain)
   203  	if err != nil {
   204  		return err
   205  	}
   206  	log := inst.Logger().WithNamespace("migration")
   207  
   208  	var docs []*vfs.FileDoc
   209  	req := &couchdb.FindRequest{
   210  		UseIndex: "by-mime-updated-at",
   211  		Selector: mango.And(
   212  			mango.Equal("mime", "text/markdown"),
   213  			mango.Exists("updated_at"),
   214  		),
   215  		Limit: 1000,
   216  	}
   217  	_, err = couchdb.FindDocsRaw(inst, consts.Files, req, &docs)
   218  	if err != nil {
   219  		return err
   220  	}
   221  	log.Infof("Found %d markdown files", len(docs))
   222  	for _, doc := range docs {
   223  		if _, ok := doc.Metadata["version"]; !ok {
   224  			log.Infof("Skip file %#v", doc)
   225  			continue
   226  		}
   227  		if err := note.Update(inst, doc.ID()); err != nil {
   228  			log.Warnf("Cannot change mime-type for note %s: %s", doc.ID(), err)
   229  		}
   230  	}
   231  
   232  	return nil
   233  }
   234  
   235  func migrateToSwiftV3(domain string) error {
   236  	c := config.GetSwiftConnection()
   237  	inst, err := instance.Get(domain)
   238  	if err != nil {
   239  		return err
   240  	}
   241  	log := inst.Logger().WithNamespace("migration")
   242  
   243  	var srcContainer, migratedFrom string
   244  	switch inst.SwiftLayout {
   245  	case 0: // layout v1
   246  		srcContainer = swiftV1ContainerPrefixCozy + inst.DBPrefix()
   247  		migratedFrom = "v1"
   248  	case 1: // layout v2
   249  		srcContainer = swiftV2ContainerPrefixCozy + inst.DBPrefix()
   250  		switch inst.DBPrefix() {
   251  		case inst.Domain:
   252  			migratedFrom = "v2a"
   253  		case inst.Prefix:
   254  			migratedFrom = "v2b"
   255  		default:
   256  			return instance.ErrInvalidSwiftLayout
   257  		}
   258  	case 2: // layout v3
   259  		return nil // Nothing to do!
   260  	default:
   261  		return instance.ErrInvalidSwiftLayout
   262  	}
   263  
   264  	log.Infof("Migrating from swift layout %s to swift layout v3", migratedFrom)
   265  
   266  	vfs := inst.VFS()
   267  	root, err := vfs.DirByID(consts.RootDirID)
   268  	if err != nil {
   269  		return err
   270  	}
   271  
   272  	mutex := config.Lock().LongOperation(inst, "vfs")
   273  	if err = mutex.Lock(); err != nil {
   274  		return err
   275  	}
   276  	defer mutex.Unlock()
   277  
   278  	ctx := context.Background()
   279  	dstContainer := swiftV3ContainerPrefix + inst.DBPrefix()
   280  	if _, _, err = c.Container(ctx, dstContainer); !errors.Is(err, swift.ContainerNotFound) {
   281  		log.Errorf("Destination container %s already exists or something went wrong. Migration canceled.", dstContainer)
   282  		return errors.New("Destination container busy")
   283  	}
   284  	if err = c.ContainerCreate(ctx, dstContainer, nil); err != nil {
   285  		return err
   286  	}
   287  	defer func() {
   288  		if err != nil {
   289  			if err := vfsswift.DeleteContainer(ctx, c, dstContainer); err != nil {
   290  				log.Errorf("Failed to delete v3 container %s: %s", dstContainer, err)
   291  			}
   292  		}
   293  	}()
   294  
   295  	if err = copyTheFilesToSwiftV3(inst, ctx, c, root, srcContainer, dstContainer); err != nil {
   296  		return err
   297  	}
   298  
   299  	meta := &swift.Metadata{"cozy-migrated-from": migratedFrom}
   300  	_ = c.ContainerUpdate(ctx, dstContainer, meta.ContainerHeaders())
   301  	if in, err := instance.Get(domain); err == nil {
   302  		inst = in
   303  	}
   304  	inst.SwiftLayout = 2
   305  	if err = instance.Update(inst); err != nil {
   306  		return err
   307  	}
   308  
   309  	// Migration done. Now clean-up oldies.
   310  
   311  	// WARNING: Don't call `err` any error below in this function or the defer func
   312  	//          will delete the new container even if the migration was successful
   313  
   314  	if deleteErr := vfs.Delete(); deleteErr != nil {
   315  		log.Errorf("Failed to delete old %s containers: %s", migratedFrom, deleteErr)
   316  	}
   317  	return nil
   318  }
   319  
   320  func copyTheFilesToSwiftV3(inst *instance.Instance, ctx context.Context, c *swift.Connection, root *vfs.DirDoc, src, dst string) error {
   321  	log := logger.WithDomain(inst.Domain).
   322  		WithNamespace("migration")
   323  	sem := semaphore.NewWeighted(maxSimultaneousCalls)
   324  	g, ctx := errgroup.WithContext(context.Background())
   325  	fs := inst.VFS()
   326  
   327  	var thumbsContainer string
   328  	switch inst.SwiftLayout {
   329  	case 0: // layout v1
   330  		thumbsContainer = swiftV1ContainerPrefixData + inst.Domain
   331  	case 1: // layout v2
   332  		thumbsContainer = swiftV2ContainerPrefixData + inst.DBPrefix()
   333  	default:
   334  		return instance.ErrInvalidSwiftLayout
   335  	}
   336  
   337  	errw := vfs.WalkAlreadyLocked(fs, root, func(_ string, d *vfs.DirDoc, f *vfs.FileDoc, err error) error {
   338  		if err != nil {
   339  			return err
   340  		}
   341  		if f == nil {
   342  			return nil
   343  		}
   344  		srcName := getSrcName(inst, f)
   345  		dstName := getDstName(inst, f)
   346  		if srcName == "" || dstName == "" {
   347  			return fmt.Errorf("Unexpected copy: %q -> %q", srcName, dstName)
   348  		}
   349  
   350  		if err := sem.Acquire(ctx, 1); err != nil {
   351  			return err
   352  		}
   353  		g.Go(func() error {
   354  			defer sem.Release(1)
   355  			err := utils.RetryWithExpBackoff(3, 200*time.Millisecond, func() error {
   356  				_, err := c.ObjectCopy(ctx, src, srcName, dst, dstName, nil)
   357  				return err
   358  			})
   359  			if err != nil {
   360  				log.Warnf("Cannot copy file from %s %s to %s %s: %s",
   361  					src, srcName, dst, dstName, err)
   362  			}
   363  			return err
   364  		})
   365  
   366  		// Copy the thumbnails
   367  		if f.Class == "image" {
   368  			srcTiny, srcSmall, srcMedium, srcLarge := getThumbsSrcNames(inst, f)
   369  			dstTiny, dstSmall, dstMedium, dstLarge := getThumbsDstNames(inst, f)
   370  			if err := sem.Acquire(ctx, 1); err != nil {
   371  				return err
   372  			}
   373  			g.Go(func() error {
   374  				defer sem.Release(1)
   375  				_, err := c.ObjectCopy(ctx, thumbsContainer, srcSmall, dst, dstSmall, nil)
   376  				if err != nil {
   377  					log.Infof("Cannot copy thumbnail tiny from %s %s to %s %s: %s",
   378  						thumbsContainer, srcTiny, dst, dstTiny, err)
   379  				}
   380  				_, err = c.ObjectCopy(ctx, thumbsContainer, srcSmall, dst, dstSmall, nil)
   381  				if err != nil {
   382  					log.Infof("Cannot copy thumbnail small from %s %s to %s %s: %s",
   383  						thumbsContainer, srcSmall, dst, dstSmall, err)
   384  				}
   385  				_, err = c.ObjectCopy(ctx, thumbsContainer, srcMedium, dst, dstMedium, nil)
   386  				if err != nil {
   387  					log.Infof("Cannot copy thumbnail medium from %s %s to %s %s: %s",
   388  						thumbsContainer, srcMedium, dst, dstMedium, err)
   389  				}
   390  				_, err = c.ObjectCopy(ctx, thumbsContainer, srcLarge, dst, dstLarge, nil)
   391  				if err != nil {
   392  					log.Infof("Cannot copy thumbnail large from %s %s to %s %s: %s",
   393  						thumbsContainer, srcLarge, dst, dstLarge, err)
   394  				}
   395  				return nil
   396  			})
   397  		}
   398  		return nil
   399  	})
   400  
   401  	if err := g.Wait(); err != nil {
   402  		return err
   403  	}
   404  	return errw
   405  }
   406  
   407  func getSrcName(inst *instance.Instance, f *vfs.FileDoc) string {
   408  	srcName := ""
   409  	switch inst.SwiftLayout {
   410  	case 0: // layout v1
   411  		srcName = f.DirID + "/" + f.DocName
   412  	case 1: // layout v2
   413  		srcName = vfsswift.MakeObjectName(f.DocID)
   414  	}
   415  	return srcName
   416  }
   417  
   418  // XXX the f FileDoc can be modified to add an InternalID
   419  func getDstName(inst *instance.Instance, f *vfs.FileDoc) string {
   420  	if f.InternalID == "" {
   421  		old := f.Clone().(*vfs.FileDoc)
   422  		f.InternalID = vfsswift.NewInternalID()
   423  		if err := couchdb.UpdateDocWithOld(inst, f, old); err != nil {
   424  			return ""
   425  		}
   426  	}
   427  	return vfsswift.MakeObjectNameV3(f.DocID, f.InternalID)
   428  }
   429  
   430  func getThumbsSrcNames(inst *instance.Instance, f *vfs.FileDoc) (string, string, string, string) {
   431  	var tiny, small, medium, large string
   432  	switch inst.SwiftLayout {
   433  	case 0: // layout v1
   434  		tiny = fmt.Sprintf("thumbs/%s-tiny", f.DocID)
   435  		small = fmt.Sprintf("thumbs/%s-small", f.DocID)
   436  		medium = fmt.Sprintf("thumbs/%s-medium", f.DocID)
   437  		large = fmt.Sprintf("thumbs/%s-large", f.DocID)
   438  	case 1: // layout v2
   439  		obj := vfsswift.MakeObjectName(f.DocID)
   440  		tiny = fmt.Sprintf("thumbs/%s-tiny", obj)
   441  		small = fmt.Sprintf("thumbs/%s-small", obj)
   442  		medium = fmt.Sprintf("thumbs/%s-medium", obj)
   443  		large = fmt.Sprintf("thumbs/%s-large", obj)
   444  	}
   445  	return tiny, small, medium, large
   446  }
   447  
   448  func getThumbsDstNames(inst *instance.Instance, f *vfs.FileDoc) (string, string, string, string) {
   449  	obj := vfsswift.MakeObjectName(f.DocID)
   450  	tiny := fmt.Sprintf("thumbs/%s-tiny", obj)
   451  	small := fmt.Sprintf("thumbs/%s-small", obj)
   452  	medium := fmt.Sprintf("thumbs/%s-medium", obj)
   453  	large := fmt.Sprintf("thumbs/%s-large", obj)
   454  	return tiny, small, medium, large
   455  }