github.com/cozy/cozy-stack@v0.0.0-20240603063001-31110fa4cae1/model/move/importer.go (about)

     1  package move
     2  
     3  import (
     4  	"archive/zip"
     5  	"encoding/json"
     6  	"errors"
     7  	"io"
     8  	"net/http"
     9  	"net/url"
    10  	"os"
    11  	"strings"
    12  	"time"
    13  
    14  	"github.com/cozy/cozy-stack/model/app"
    15  	"github.com/cozy/cozy-stack/model/contact"
    16  	"github.com/cozy/cozy-stack/model/instance"
    17  	"github.com/cozy/cozy-stack/model/job"
    18  	"github.com/cozy/cozy-stack/model/permission"
    19  	"github.com/cozy/cozy-stack/model/sharing"
    20  	"github.com/cozy/cozy-stack/model/vfs"
    21  	"github.com/cozy/cozy-stack/pkg/consts"
    22  	"github.com/cozy/cozy-stack/pkg/couchdb"
    23  	"github.com/cozy/cozy-stack/pkg/safehttp"
    24  	multierror "github.com/hashicorp/go-multierror"
    25  )
    26  
    27  type importer struct {
    28  	inst            *instance.Instance
    29  	fs              vfs.VFS
    30  	options         ImportOptions
    31  	doc             *ExportDoc
    32  	servicesInError map[string]bool // a map, not a slice, to have unique values
    33  	tmpFile         string
    34  	doctype         string
    35  	docs            []interface{}
    36  	triggers        []*job.TriggerInfos
    37  }
    38  
    39  func (im *importer) importPart(cursor string) error {
    40  	defer func() {
    41  		if im.tmpFile != "" {
    42  			if err := os.Remove(im.tmpFile); err != nil {
    43  				im.inst.Logger().WithNamespace("move").
    44  					Warnf("Cannot remove temp file %s: %s", im.tmpFile, err)
    45  			}
    46  		}
    47  	}()
    48  	if err := im.downloadFile(cursor); err != nil {
    49  		return err
    50  	}
    51  	zr, err := zip.OpenReader(im.tmpFile)
    52  	if err != nil {
    53  		return err
    54  	}
    55  	err = im.importZip(&zr.Reader)
    56  	if errc := zr.Close(); err == nil {
    57  		err = errc
    58  	}
    59  	return err
    60  }
    61  
    62  func (im *importer) downloadFile(cursor string) error {
    63  	u, err := url.Parse(im.options.ManifestURL)
    64  	if err != nil {
    65  		return err
    66  	}
    67  	u.Path = strings.Replace(u.Path, "/move/exports/", "/move/exports/data/", 1)
    68  	if cursor != "" {
    69  		u.RawQuery = url.Values{"cursor": {cursor}}.Encode()
    70  	}
    71  	res, err := safehttp.ClientWithKeepAlive.Get(u.String())
    72  	if err != nil {
    73  		return err
    74  	}
    75  	defer res.Body.Close()
    76  	if res.StatusCode != http.StatusOK {
    77  		return ErrExportNotFound
    78  	}
    79  	f, err := os.CreateTemp("", "export-*")
    80  	if err != nil {
    81  		return err
    82  	}
    83  	im.tmpFile = f.Name()
    84  	_, err = io.Copy(f, res.Body)
    85  	if errc := f.Close(); err == nil {
    86  		err = errc
    87  	}
    88  	return err
    89  }
    90  
    91  func (im *importer) importZip(zr *zip.Reader) error {
    92  	var errm error
    93  
    94  	for i, file := range zr.File {
    95  		if !strings.HasPrefix(file.FileHeader.Name, ExportDataDir+"/") {
    96  			continue
    97  		}
    98  		name := strings.TrimPrefix(file.FileHeader.Name, ExportDataDir+"/")
    99  		parts := strings.SplitN(name, "/", 2)
   100  		if len(parts) != 2 {
   101  			continue // "instance.json" for example
   102  		}
   103  		doctype := parts[0]
   104  		id := strings.TrimSuffix(parts[1], ".json")
   105  
   106  		// Special cases
   107  		switch doctype {
   108  		case consts.Exports:
   109  			// Importing exports would just be a mess, so skip them
   110  			continue
   111  		case consts.Sessions:
   112  			// We don't want to import the sessions from another instance
   113  			continue
   114  		case consts.BitwardenCiphers, consts.BitwardenFolders, consts.BitwardenProfiles,
   115  			consts.BitwardenOrganizations, consts.BitwardenContacts:
   116  			// Bitwarden documents are encypted E2E, so they cannot be imported
   117  			// as raw documents
   118  			continue
   119  		case consts.Sharings:
   120  			// Sharings are imported only for a move
   121  			if im.options.MoveFrom == nil {
   122  				continue
   123  			}
   124  			if err := im.importSharing(file); err != nil {
   125  				errm = multierror.Append(errm, err)
   126  			}
   127  			continue
   128  		case consts.Shared:
   129  			if im.options.MoveFrom == nil {
   130  				continue
   131  			}
   132  		case consts.Permissions:
   133  			if im.options.MoveFrom == nil {
   134  				continue
   135  			}
   136  			if err := im.importPermission(file); err != nil {
   137  				errm = multierror.Append(errm, err)
   138  			}
   139  			continue
   140  		case consts.Settings:
   141  			// Keep the email, public name and stuff related to the cloudery
   142  			// from the destination Cozy. Same for the bitwarden settings
   143  			// derived from the passphrase.
   144  			if id == consts.InstanceSettingsID || id == consts.BitwardenSettingsID {
   145  				continue
   146  			}
   147  		case consts.Apps, consts.Konnectors:
   148  			im.installApp(id)
   149  			continue
   150  		case consts.Accounts:
   151  			if err := im.importAccount(file); err != nil {
   152  				errm = multierror.Append(errm, err)
   153  			}
   154  			continue
   155  		case consts.Triggers:
   156  			if err := im.importTrigger(file); err != nil {
   157  				errm = multierror.Append(errm, err)
   158  			}
   159  			continue
   160  		case consts.Files:
   161  			var content *zip.File
   162  			if i < len(zr.File)-1 {
   163  				content = zr.File[i+1]
   164  			}
   165  			if err := im.importFile(file, content); err != nil {
   166  				errm = multierror.Append(errm, err)
   167  			}
   168  			continue
   169  		case consts.FilesVersions:
   170  			if i >= len(zr.File)-1 {
   171  				continue
   172  			}
   173  			if err := im.importFileVersion(file, zr.File[i+1]); err != nil {
   174  				errm = multierror.Append(errm, err)
   175  			}
   176  			continue
   177  		}
   178  
   179  		// Normal documents
   180  		if doctype != im.doctype || len(im.docs) >= 100 {
   181  			if err := im.flush(); err != nil {
   182  				errm = multierror.Append(errm, err)
   183  				im.docs = nil
   184  				im.doctype = ""
   185  			}
   186  		}
   187  		doc, err := im.readDoc(file)
   188  		if err != nil {
   189  			errm = multierror.Append(errm, err)
   190  			continue
   191  		}
   192  		delete(doc, "_rev")
   193  		im.doctype = doctype
   194  		im.docs = append(im.docs, doc)
   195  	}
   196  
   197  	if err := im.flush(); err != nil {
   198  		errm = multierror.Append(errm, err)
   199  	}
   200  
   201  	// Import the triggers at the end to avoid creating many jobs when
   202  	// importing the files.
   203  	if err := im.importTriggers(); err != nil {
   204  		errm = multierror.Append(errm, err)
   205  	}
   206  
   207  	// Reinject the email address from the destination Cozy in the myself
   208  	// contact document
   209  	if myself, err := contact.GetMyself(im.inst); err == nil {
   210  		if email, err := im.inst.SettingsEMail(); err == nil && email != "" {
   211  			addr, _ := myself.ToMailAddress()
   212  			if addr == nil || addr.Email != email {
   213  				myself.JSONDoc.M["email"] = []map[string]interface{}{
   214  					{
   215  						"address": email,
   216  						"primary": true,
   217  					},
   218  				}
   219  				_ = couchdb.UpdateDoc(im.inst, myself)
   220  			}
   221  		}
   222  	}
   223  	return errm
   224  }
   225  
   226  func (im *importer) flush() error {
   227  	if len(im.docs) == 0 {
   228  		return nil
   229  	}
   230  
   231  	olds := make([]interface{}, len(im.docs))
   232  	if err := couchdb.BulkUpdateDocs(im.inst, im.doctype, im.docs, olds); err != nil {
   233  		// XXX CouchDB can be overloaded sometimes when importing lots of documents.
   234  		// Let's wait a bit and retry...
   235  		for i := 0; i < 12; i++ {
   236  			time.Sleep(5 * time.Minute)
   237  			err = couchdb.BulkUpdateDocs(im.inst, im.doctype, im.docs, olds)
   238  			if err == nil {
   239  				break
   240  			}
   241  		}
   242  		if err != nil {
   243  			return err
   244  		}
   245  	}
   246  	// XXX Not too fast, CouchDB can be easily overloaded...
   247  	time.Sleep(100 * time.Millisecond)
   248  
   249  	im.doctype = ""
   250  	im.docs = nil
   251  	return nil
   252  }
   253  
   254  func (im *importer) readDoc(zf *zip.File) (map[string]interface{}, error) {
   255  	r, err := zf.Open()
   256  	if err != nil {
   257  		return nil, err
   258  	}
   259  	var doc map[string]interface{}
   260  	err = json.NewDecoder(r).Decode(&doc)
   261  	if errc := r.Close(); errc != nil {
   262  		return nil, errc
   263  	}
   264  	if err != nil {
   265  		return nil, err
   266  	}
   267  	return doc, nil
   268  }
   269  
   270  func (im *importer) importAccount(zf *zip.File) error {
   271  	doc, err := im.readDoc(zf)
   272  	if err != nil {
   273  		return err
   274  	}
   275  
   276  	// Note: the slug will be empty for aggregator accounts, and it won't be
   277  	// imported as an aggregator account is used by other accounts with a hook
   278  	// on deletion.
   279  	slug, _ := doc["account_type"].(string)
   280  	man, err := app.GetKonnectorBySlug(im.inst, slug)
   281  	if errors.Is(err, app.ErrNotFound) {
   282  		im.installApp(consts.Konnectors + "/" + slug)
   283  		man, err = app.GetKonnectorBySlug(im.inst, slug)
   284  	}
   285  	if err != nil || man.OnDeleteAccount() != "" {
   286  		im.servicesInError[slug] = true
   287  		return nil
   288  	}
   289  
   290  	docs := []interface{}{doc}
   291  	olds := make([]interface{}, len(docs))
   292  	if err := couchdb.EnsureDBExist(im.inst, consts.Accounts); err != nil {
   293  		return err
   294  	}
   295  	return couchdb.BulkUpdateDocs(im.inst, consts.Accounts, docs, olds)
   296  }
   297  
   298  func (im *importer) readTrigger(zf *zip.File) (*job.TriggerInfos, error) {
   299  	r, err := zf.Open()
   300  	if err != nil {
   301  		return nil, err
   302  	}
   303  	doc := &job.TriggerInfos{}
   304  	err = json.NewDecoder(r).Decode(doc)
   305  	if errc := r.Close(); errc != nil {
   306  		return nil, errc
   307  	}
   308  	if err != nil {
   309  		return nil, err
   310  	}
   311  	return doc, nil
   312  }
   313  
   314  func (im *importer) importTrigger(zf *zip.File) error {
   315  	doc, err := im.readTrigger(zf)
   316  	if err != nil {
   317  		return err
   318  	}
   319  	switch doc.WorkerType {
   320  	case "share-track", "share-replicate", "share-upload":
   321  		// The share-* triggers are imported only for a move
   322  		if im.options.MoveFrom == nil {
   323  			return nil
   324  		}
   325  	case "konnector":
   326  		// OK, import it
   327  	default:
   328  		return nil
   329  	}
   330  	// We don't import triggers now, but wait after files has been imported to
   331  	// avoid creating many jobs when importing shared files.
   332  	im.triggers = append(im.triggers, doc)
   333  	return nil
   334  }
   335  
   336  func (im *importer) importTriggers() error {
   337  	var errm error
   338  	for _, doc := range im.triggers {
   339  		doc.SetRev("")
   340  		t, err := job.NewTrigger(im.inst, *doc, nil)
   341  		if err != nil {
   342  			errm = multierror.Append(errm, err)
   343  			continue
   344  		}
   345  		if err = job.System().AddTrigger(t); err != nil && !couchdb.IsConflictError(err) {
   346  			errm = multierror.Append(errm, err)
   347  		}
   348  	}
   349  	return errm
   350  }
   351  
   352  func (im *importer) installApp(id string) {
   353  	parts := strings.SplitN(id, "/", 2)
   354  	if len(parts) != 2 {
   355  		return
   356  	}
   357  	doctype := parts[0]
   358  	apptype := consts.WebappType
   359  	if doctype == consts.Konnectors {
   360  		apptype = consts.KonnectorType
   361  	}
   362  	slug := parts[1]
   363  	source := "registry://" + slug + "/stable"
   364  
   365  	installer, err := app.NewInstaller(im.inst, app.Copier(apptype, im.inst),
   366  		&app.InstallerOptions{
   367  			Operation:  app.Install,
   368  			Type:       apptype,
   369  			SourceURL:  source,
   370  			Slug:       slug,
   371  			Registries: im.inst.Registries(),
   372  		},
   373  	)
   374  	if err == nil {
   375  		_, err = installer.RunSync()
   376  	}
   377  	if err != nil && !errors.Is(err, app.ErrAlreadyExists) {
   378  		im.servicesInError[slug] = true
   379  	}
   380  }
   381  
   382  func (im *importer) importFile(zdoc, zcontent *zip.File) error {
   383  	doc, err := im.readFileDoc(zdoc)
   384  	if err != nil {
   385  		return err
   386  	}
   387  	dirDoc, fileDoc := doc.Refine()
   388  	if dirDoc != nil {
   389  		dirDoc.SetRev("")
   390  		if dirDoc.DocID == consts.RootDirID || dirDoc.DocID == consts.TrashDirID {
   391  			return nil
   392  		}
   393  		return im.fs.CreateDir(dirDoc)
   394  	}
   395  
   396  	if zcontent == nil {
   397  		return errors.New("No content for file")
   398  	}
   399  	fileDoc.SetRev("")
   400  	// Do not trust carbon copy and electronic safe flags on import
   401  	if fileDoc.Metadata != nil {
   402  		delete(fileDoc.Metadata, consts.CarbonCopyKey)
   403  		delete(fileDoc.Metadata, consts.ElectronicSafeKey)
   404  	}
   405  	f, err := im.fs.CreateFile(fileDoc, nil, vfs.AllowCreationInTrash)
   406  	if err != nil {
   407  		return err
   408  	}
   409  
   410  	content, err := zcontent.Open()
   411  	if err != nil {
   412  		return err
   413  	}
   414  	_, err = io.Copy(f, content)
   415  	if errc := f.Close(); err == nil {
   416  		err = errc
   417  	}
   418  	return err
   419  }
   420  
   421  func (im *importer) readFileDoc(zf *zip.File) (*vfs.DirOrFileDoc, error) {
   422  	r, err := zf.Open()
   423  	if err != nil {
   424  		return nil, err
   425  	}
   426  	var doc vfs.DirOrFileDoc
   427  	err = json.NewDecoder(r).Decode(&doc)
   428  	if errc := r.Close(); errc != nil {
   429  		return nil, errc
   430  	}
   431  	if err != nil {
   432  		return nil, err
   433  	}
   434  	return &doc, nil
   435  }
   436  
   437  func (im *importer) importFileVersion(zdoc, zcontent *zip.File) error {
   438  	doc, err := im.readVersion(zdoc)
   439  	if err != nil {
   440  		return err
   441  	}
   442  	content, err := zcontent.Open()
   443  	if err != nil {
   444  		return err
   445  	}
   446  	doc.SetRev("")
   447  	return im.fs.ImportFileVersion(doc, content)
   448  }
   449  
   450  func (im *importer) readVersion(zf *zip.File) (*vfs.Version, error) {
   451  	r, err := zf.Open()
   452  	if err != nil {
   453  		return nil, err
   454  	}
   455  	var doc vfs.Version
   456  	err = json.NewDecoder(r).Decode(&doc)
   457  	if errc := r.Close(); errc != nil {
   458  		return nil, errc
   459  	}
   460  	if err != nil {
   461  		return nil, err
   462  	}
   463  	return &doc, nil
   464  }
   465  
   466  func (im *importer) importSharing(zf *zip.File) error {
   467  	s, err := im.readSharing(zf)
   468  	if err != nil {
   469  		return err
   470  	}
   471  	// XXX Do not import sharing for bitwarden stuff
   472  	if s.FirstBitwardenOrganizationRule() != nil {
   473  		return nil
   474  	}
   475  	s.Initial = false
   476  	s.NbFiles = 0
   477  	s.UpdatedAt = time.Now()
   478  	s.SetRev("")
   479  	if s.Owner {
   480  		s.MovedFrom = s.Members[0].Instance
   481  		s.Members[0].Instance = im.inst.PageURL("", nil)
   482  	} else {
   483  		targetURL := strings.TrimSuffix(im.options.MoveFrom.URL, "/")
   484  		for i, m := range s.Members {
   485  			if m.Instance == targetURL {
   486  				s.MovedFrom = s.Members[i].Instance
   487  				s.Members[i].Instance = im.inst.PageURL("", nil)
   488  			}
   489  		}
   490  	}
   491  	return couchdb.CreateNamedDoc(im.inst, s)
   492  }
   493  
   494  func (im *importer) readSharing(zf *zip.File) (*sharing.Sharing, error) {
   495  	r, err := zf.Open()
   496  	if err != nil {
   497  		return nil, err
   498  	}
   499  	doc := &sharing.Sharing{}
   500  	err = json.NewDecoder(r).Decode(doc)
   501  	if errc := r.Close(); errc != nil {
   502  		return nil, errc
   503  	}
   504  	if err != nil {
   505  		return nil, err
   506  	}
   507  	return doc, nil
   508  }
   509  
   510  func (im *importer) importPermission(zf *zip.File) error {
   511  	doc, err := im.readPermission(zf)
   512  	if err != nil {
   513  		return err
   514  	}
   515  	// We only import permission documents for sharings
   516  	if doc.Type != permission.TypeShareByLink && doc.Type != permission.TypeSharePreview {
   517  		return nil
   518  	}
   519  	// We need to remake the long codes with the new instance domain
   520  	for name := range doc.Codes {
   521  		longcode, err := im.inst.CreateShareCode(name)
   522  		if err != nil {
   523  			return err
   524  		}
   525  		doc.Codes[name] = longcode
   526  	}
   527  	doc.SetRev("")
   528  	return couchdb.CreateNamedDoc(im.inst, doc)
   529  }
   530  
   531  func (im *importer) readPermission(zf *zip.File) (*permission.Permission, error) {
   532  	r, err := zf.Open()
   533  	if err != nil {
   534  		return nil, err
   535  	}
   536  	doc := &permission.Permission{}
   537  	err = json.NewDecoder(r).Decode(doc)
   538  	if errc := r.Close(); errc != nil {
   539  		return nil, errc
   540  	}
   541  	if err != nil {
   542  		return nil, err
   543  	}
   544  	return doc, nil
   545  }