github.com/cozy/cozy-stack@v0.0.0-20240603063001-31110fa4cae1/model/move/importer.go (about) 1 package move 2 3 import ( 4 "archive/zip" 5 "encoding/json" 6 "errors" 7 "io" 8 "net/http" 9 "net/url" 10 "os" 11 "strings" 12 "time" 13 14 "github.com/cozy/cozy-stack/model/app" 15 "github.com/cozy/cozy-stack/model/contact" 16 "github.com/cozy/cozy-stack/model/instance" 17 "github.com/cozy/cozy-stack/model/job" 18 "github.com/cozy/cozy-stack/model/permission" 19 "github.com/cozy/cozy-stack/model/sharing" 20 "github.com/cozy/cozy-stack/model/vfs" 21 "github.com/cozy/cozy-stack/pkg/consts" 22 "github.com/cozy/cozy-stack/pkg/couchdb" 23 "github.com/cozy/cozy-stack/pkg/safehttp" 24 multierror "github.com/hashicorp/go-multierror" 25 ) 26 27 type importer struct { 28 inst *instance.Instance 29 fs vfs.VFS 30 options ImportOptions 31 doc *ExportDoc 32 servicesInError map[string]bool // a map, not a slice, to have unique values 33 tmpFile string 34 doctype string 35 docs []interface{} 36 triggers []*job.TriggerInfos 37 } 38 39 func (im *importer) importPart(cursor string) error { 40 defer func() { 41 if im.tmpFile != "" { 42 if err := os.Remove(im.tmpFile); err != nil { 43 im.inst.Logger().WithNamespace("move"). 44 Warnf("Cannot remove temp file %s: %s", im.tmpFile, err) 45 } 46 } 47 }() 48 if err := im.downloadFile(cursor); err != nil { 49 return err 50 } 51 zr, err := zip.OpenReader(im.tmpFile) 52 if err != nil { 53 return err 54 } 55 err = im.importZip(&zr.Reader) 56 if errc := zr.Close(); err == nil { 57 err = errc 58 } 59 return err 60 } 61 62 func (im *importer) downloadFile(cursor string) error { 63 u, err := url.Parse(im.options.ManifestURL) 64 if err != nil { 65 return err 66 } 67 u.Path = strings.Replace(u.Path, "/move/exports/", "/move/exports/data/", 1) 68 if cursor != "" { 69 u.RawQuery = url.Values{"cursor": {cursor}}.Encode() 70 } 71 res, err := safehttp.ClientWithKeepAlive.Get(u.String()) 72 if err != nil { 73 return err 74 } 75 defer res.Body.Close() 76 if res.StatusCode != http.StatusOK { 77 return ErrExportNotFound 78 } 79 f, err := os.CreateTemp("", "export-*") 80 if err != nil { 81 return err 82 } 83 im.tmpFile = f.Name() 84 _, err = io.Copy(f, res.Body) 85 if errc := f.Close(); err == nil { 86 err = errc 87 } 88 return err 89 } 90 91 func (im *importer) importZip(zr *zip.Reader) error { 92 var errm error 93 94 for i, file := range zr.File { 95 if !strings.HasPrefix(file.FileHeader.Name, ExportDataDir+"/") { 96 continue 97 } 98 name := strings.TrimPrefix(file.FileHeader.Name, ExportDataDir+"/") 99 parts := strings.SplitN(name, "/", 2) 100 if len(parts) != 2 { 101 continue // "instance.json" for example 102 } 103 doctype := parts[0] 104 id := strings.TrimSuffix(parts[1], ".json") 105 106 // Special cases 107 switch doctype { 108 case consts.Exports: 109 // Importing exports would just be a mess, so skip them 110 continue 111 case consts.Sessions: 112 // We don't want to import the sessions from another instance 113 continue 114 case consts.BitwardenCiphers, consts.BitwardenFolders, consts.BitwardenProfiles, 115 consts.BitwardenOrganizations, consts.BitwardenContacts: 116 // Bitwarden documents are encypted E2E, so they cannot be imported 117 // as raw documents 118 continue 119 case consts.Sharings: 120 // Sharings are imported only for a move 121 if im.options.MoveFrom == nil { 122 continue 123 } 124 if err := im.importSharing(file); err != nil { 125 errm = multierror.Append(errm, err) 126 } 127 continue 128 case consts.Shared: 129 if im.options.MoveFrom == nil { 130 continue 131 } 132 case consts.Permissions: 133 if im.options.MoveFrom == nil { 134 continue 135 } 136 if err := im.importPermission(file); err != nil { 137 errm = multierror.Append(errm, err) 138 } 139 continue 140 case consts.Settings: 141 // Keep the email, public name and stuff related to the cloudery 142 // from the destination Cozy. Same for the bitwarden settings 143 // derived from the passphrase. 144 if id == consts.InstanceSettingsID || id == consts.BitwardenSettingsID { 145 continue 146 } 147 case consts.Apps, consts.Konnectors: 148 im.installApp(id) 149 continue 150 case consts.Accounts: 151 if err := im.importAccount(file); err != nil { 152 errm = multierror.Append(errm, err) 153 } 154 continue 155 case consts.Triggers: 156 if err := im.importTrigger(file); err != nil { 157 errm = multierror.Append(errm, err) 158 } 159 continue 160 case consts.Files: 161 var content *zip.File 162 if i < len(zr.File)-1 { 163 content = zr.File[i+1] 164 } 165 if err := im.importFile(file, content); err != nil { 166 errm = multierror.Append(errm, err) 167 } 168 continue 169 case consts.FilesVersions: 170 if i >= len(zr.File)-1 { 171 continue 172 } 173 if err := im.importFileVersion(file, zr.File[i+1]); err != nil { 174 errm = multierror.Append(errm, err) 175 } 176 continue 177 } 178 179 // Normal documents 180 if doctype != im.doctype || len(im.docs) >= 100 { 181 if err := im.flush(); err != nil { 182 errm = multierror.Append(errm, err) 183 im.docs = nil 184 im.doctype = "" 185 } 186 } 187 doc, err := im.readDoc(file) 188 if err != nil { 189 errm = multierror.Append(errm, err) 190 continue 191 } 192 delete(doc, "_rev") 193 im.doctype = doctype 194 im.docs = append(im.docs, doc) 195 } 196 197 if err := im.flush(); err != nil { 198 errm = multierror.Append(errm, err) 199 } 200 201 // Import the triggers at the end to avoid creating many jobs when 202 // importing the files. 203 if err := im.importTriggers(); err != nil { 204 errm = multierror.Append(errm, err) 205 } 206 207 // Reinject the email address from the destination Cozy in the myself 208 // contact document 209 if myself, err := contact.GetMyself(im.inst); err == nil { 210 if email, err := im.inst.SettingsEMail(); err == nil && email != "" { 211 addr, _ := myself.ToMailAddress() 212 if addr == nil || addr.Email != email { 213 myself.JSONDoc.M["email"] = []map[string]interface{}{ 214 { 215 "address": email, 216 "primary": true, 217 }, 218 } 219 _ = couchdb.UpdateDoc(im.inst, myself) 220 } 221 } 222 } 223 return errm 224 } 225 226 func (im *importer) flush() error { 227 if len(im.docs) == 0 { 228 return nil 229 } 230 231 olds := make([]interface{}, len(im.docs)) 232 if err := couchdb.BulkUpdateDocs(im.inst, im.doctype, im.docs, olds); err != nil { 233 // XXX CouchDB can be overloaded sometimes when importing lots of documents. 234 // Let's wait a bit and retry... 235 for i := 0; i < 12; i++ { 236 time.Sleep(5 * time.Minute) 237 err = couchdb.BulkUpdateDocs(im.inst, im.doctype, im.docs, olds) 238 if err == nil { 239 break 240 } 241 } 242 if err != nil { 243 return err 244 } 245 } 246 // XXX Not too fast, CouchDB can be easily overloaded... 247 time.Sleep(100 * time.Millisecond) 248 249 im.doctype = "" 250 im.docs = nil 251 return nil 252 } 253 254 func (im *importer) readDoc(zf *zip.File) (map[string]interface{}, error) { 255 r, err := zf.Open() 256 if err != nil { 257 return nil, err 258 } 259 var doc map[string]interface{} 260 err = json.NewDecoder(r).Decode(&doc) 261 if errc := r.Close(); errc != nil { 262 return nil, errc 263 } 264 if err != nil { 265 return nil, err 266 } 267 return doc, nil 268 } 269 270 func (im *importer) importAccount(zf *zip.File) error { 271 doc, err := im.readDoc(zf) 272 if err != nil { 273 return err 274 } 275 276 // Note: the slug will be empty for aggregator accounts, and it won't be 277 // imported as an aggregator account is used by other accounts with a hook 278 // on deletion. 279 slug, _ := doc["account_type"].(string) 280 man, err := app.GetKonnectorBySlug(im.inst, slug) 281 if errors.Is(err, app.ErrNotFound) { 282 im.installApp(consts.Konnectors + "/" + slug) 283 man, err = app.GetKonnectorBySlug(im.inst, slug) 284 } 285 if err != nil || man.OnDeleteAccount() != "" { 286 im.servicesInError[slug] = true 287 return nil 288 } 289 290 docs := []interface{}{doc} 291 olds := make([]interface{}, len(docs)) 292 if err := couchdb.EnsureDBExist(im.inst, consts.Accounts); err != nil { 293 return err 294 } 295 return couchdb.BulkUpdateDocs(im.inst, consts.Accounts, docs, olds) 296 } 297 298 func (im *importer) readTrigger(zf *zip.File) (*job.TriggerInfos, error) { 299 r, err := zf.Open() 300 if err != nil { 301 return nil, err 302 } 303 doc := &job.TriggerInfos{} 304 err = json.NewDecoder(r).Decode(doc) 305 if errc := r.Close(); errc != nil { 306 return nil, errc 307 } 308 if err != nil { 309 return nil, err 310 } 311 return doc, nil 312 } 313 314 func (im *importer) importTrigger(zf *zip.File) error { 315 doc, err := im.readTrigger(zf) 316 if err != nil { 317 return err 318 } 319 switch doc.WorkerType { 320 case "share-track", "share-replicate", "share-upload": 321 // The share-* triggers are imported only for a move 322 if im.options.MoveFrom == nil { 323 return nil 324 } 325 case "konnector": 326 // OK, import it 327 default: 328 return nil 329 } 330 // We don't import triggers now, but wait after files has been imported to 331 // avoid creating many jobs when importing shared files. 332 im.triggers = append(im.triggers, doc) 333 return nil 334 } 335 336 func (im *importer) importTriggers() error { 337 var errm error 338 for _, doc := range im.triggers { 339 doc.SetRev("") 340 t, err := job.NewTrigger(im.inst, *doc, nil) 341 if err != nil { 342 errm = multierror.Append(errm, err) 343 continue 344 } 345 if err = job.System().AddTrigger(t); err != nil && !couchdb.IsConflictError(err) { 346 errm = multierror.Append(errm, err) 347 } 348 } 349 return errm 350 } 351 352 func (im *importer) installApp(id string) { 353 parts := strings.SplitN(id, "/", 2) 354 if len(parts) != 2 { 355 return 356 } 357 doctype := parts[0] 358 apptype := consts.WebappType 359 if doctype == consts.Konnectors { 360 apptype = consts.KonnectorType 361 } 362 slug := parts[1] 363 source := "registry://" + slug + "/stable" 364 365 installer, err := app.NewInstaller(im.inst, app.Copier(apptype, im.inst), 366 &app.InstallerOptions{ 367 Operation: app.Install, 368 Type: apptype, 369 SourceURL: source, 370 Slug: slug, 371 Registries: im.inst.Registries(), 372 }, 373 ) 374 if err == nil { 375 _, err = installer.RunSync() 376 } 377 if err != nil && !errors.Is(err, app.ErrAlreadyExists) { 378 im.servicesInError[slug] = true 379 } 380 } 381 382 func (im *importer) importFile(zdoc, zcontent *zip.File) error { 383 doc, err := im.readFileDoc(zdoc) 384 if err != nil { 385 return err 386 } 387 dirDoc, fileDoc := doc.Refine() 388 if dirDoc != nil { 389 dirDoc.SetRev("") 390 if dirDoc.DocID == consts.RootDirID || dirDoc.DocID == consts.TrashDirID { 391 return nil 392 } 393 return im.fs.CreateDir(dirDoc) 394 } 395 396 if zcontent == nil { 397 return errors.New("No content for file") 398 } 399 fileDoc.SetRev("") 400 // Do not trust carbon copy and electronic safe flags on import 401 if fileDoc.Metadata != nil { 402 delete(fileDoc.Metadata, consts.CarbonCopyKey) 403 delete(fileDoc.Metadata, consts.ElectronicSafeKey) 404 } 405 f, err := im.fs.CreateFile(fileDoc, nil, vfs.AllowCreationInTrash) 406 if err != nil { 407 return err 408 } 409 410 content, err := zcontent.Open() 411 if err != nil { 412 return err 413 } 414 _, err = io.Copy(f, content) 415 if errc := f.Close(); err == nil { 416 err = errc 417 } 418 return err 419 } 420 421 func (im *importer) readFileDoc(zf *zip.File) (*vfs.DirOrFileDoc, error) { 422 r, err := zf.Open() 423 if err != nil { 424 return nil, err 425 } 426 var doc vfs.DirOrFileDoc 427 err = json.NewDecoder(r).Decode(&doc) 428 if errc := r.Close(); errc != nil { 429 return nil, errc 430 } 431 if err != nil { 432 return nil, err 433 } 434 return &doc, nil 435 } 436 437 func (im *importer) importFileVersion(zdoc, zcontent *zip.File) error { 438 doc, err := im.readVersion(zdoc) 439 if err != nil { 440 return err 441 } 442 content, err := zcontent.Open() 443 if err != nil { 444 return err 445 } 446 doc.SetRev("") 447 return im.fs.ImportFileVersion(doc, content) 448 } 449 450 func (im *importer) readVersion(zf *zip.File) (*vfs.Version, error) { 451 r, err := zf.Open() 452 if err != nil { 453 return nil, err 454 } 455 var doc vfs.Version 456 err = json.NewDecoder(r).Decode(&doc) 457 if errc := r.Close(); errc != nil { 458 return nil, errc 459 } 460 if err != nil { 461 return nil, err 462 } 463 return &doc, nil 464 } 465 466 func (im *importer) importSharing(zf *zip.File) error { 467 s, err := im.readSharing(zf) 468 if err != nil { 469 return err 470 } 471 // XXX Do not import sharing for bitwarden stuff 472 if s.FirstBitwardenOrganizationRule() != nil { 473 return nil 474 } 475 s.Initial = false 476 s.NbFiles = 0 477 s.UpdatedAt = time.Now() 478 s.SetRev("") 479 if s.Owner { 480 s.MovedFrom = s.Members[0].Instance 481 s.Members[0].Instance = im.inst.PageURL("", nil) 482 } else { 483 targetURL := strings.TrimSuffix(im.options.MoveFrom.URL, "/") 484 for i, m := range s.Members { 485 if m.Instance == targetURL { 486 s.MovedFrom = s.Members[i].Instance 487 s.Members[i].Instance = im.inst.PageURL("", nil) 488 } 489 } 490 } 491 return couchdb.CreateNamedDoc(im.inst, s) 492 } 493 494 func (im *importer) readSharing(zf *zip.File) (*sharing.Sharing, error) { 495 r, err := zf.Open() 496 if err != nil { 497 return nil, err 498 } 499 doc := &sharing.Sharing{} 500 err = json.NewDecoder(r).Decode(doc) 501 if errc := r.Close(); errc != nil { 502 return nil, errc 503 } 504 if err != nil { 505 return nil, err 506 } 507 return doc, nil 508 } 509 510 func (im *importer) importPermission(zf *zip.File) error { 511 doc, err := im.readPermission(zf) 512 if err != nil { 513 return err 514 } 515 // We only import permission documents for sharings 516 if doc.Type != permission.TypeShareByLink && doc.Type != permission.TypeSharePreview { 517 return nil 518 } 519 // We need to remake the long codes with the new instance domain 520 for name := range doc.Codes { 521 longcode, err := im.inst.CreateShareCode(name) 522 if err != nil { 523 return err 524 } 525 doc.Codes[name] = longcode 526 } 527 doc.SetRev("") 528 return couchdb.CreateNamedDoc(im.inst, doc) 529 } 530 531 func (im *importer) readPermission(zf *zip.File) (*permission.Permission, error) { 532 r, err := zf.Open() 533 if err != nil { 534 return nil, err 535 } 536 doc := &permission.Permission{} 537 err = json.NewDecoder(r).Decode(doc) 538 if errc := r.Close(); errc != nil { 539 return nil, errc 540 } 541 if err != nil { 542 return nil, err 543 } 544 return doc, nil 545 }