go.fuchsia.dev/jiri@v0.0.0-20240502161911-b66513b29486/project/operations.go (about) 1 // Copyright 2017 The Fuchsia Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package project 6 7 import ( 8 "fmt" 9 "hash/fnv" 10 "io" 11 "io/ioutil" 12 "os" 13 "path" 14 "path/filepath" 15 "sort" 16 "strings" 17 "sync" 18 19 "go.fuchsia.dev/jiri" 20 "go.fuchsia.dev/jiri/gitutil" 21 "go.fuchsia.dev/jiri/log" 22 "go.fuchsia.dev/jiri/osutil" 23 ) 24 25 const ( 26 changeRemoteOpKind = "change-remote" 27 createOpKind = "create" 28 deleteOpKind = "delete" 29 moveOpKind = "move" 30 nullOpKind = "null" 31 updateOpKind = "update" 32 ) 33 34 type operation interface { 35 // Project identifies the project this operation pertains to. 36 Project() Project 37 // Kind returns the kind of operation. 38 Kind() string 39 // Run executes the operation. 40 Run(jirix *jiri.X) error 41 // String returns a string representation of the operation. 42 String() string 43 // Test checks whether the operation would fail. 44 Test(jirix *jiri.X) error 45 // Source returns the original path of the Project. 46 Source() string 47 // Destination returns the future path of the Project. 48 Destination() string 49 } 50 51 // commonOperation represents a project operation. 52 type commonOperation struct { 53 // project holds information about the project such as its 54 // name, local path, and the protocol it uses for version 55 // control. 56 project Project 57 // destination is the new project path. 58 destination string 59 // source is the current project path. 60 source string 61 // state is the state of the local project 62 state ProjectState 63 } 64 65 func (op commonOperation) Project() Project { 66 return op.project 67 } 68 69 func (op commonOperation) Source() string { 70 return op.source 71 } 72 73 func (op commonOperation) Destination() string { 74 return op.destination 75 } 76 77 // createOperation represents the creation of a project. 78 type createOperation struct { 79 commonOperation 80 } 81 82 func (op createOperation) Kind() string { 83 return createOpKind 84 } 85 86 func (op createOperation) checkoutProject(jirix *jiri.X, cache string) error { 87 var err error 88 remote := rewriteRemote(jirix, op.project.Remote) 89 scm := gitutil.New(jirix, gitutil.RootDirOpt(op.project.Path)) 90 // Hack to make fuchsia.git happen 91 if op.destination == jirix.Root { 92 if err = scm.Init(op.destination); err != nil { 93 return err 94 } 95 if err = scm.AddOrReplaceRemote("origin", remote); err != nil { 96 return err 97 } 98 // This appears to be set to 0 via some quirk of `git init`. 99 if err := scm.Config("core.repositoryformatversion", "1"); err != nil { 100 return err 101 } 102 if jirix.UsePartialClone(op.project.Remote) { 103 if err := scm.Config("extensions.partialClone", "origin"); err != nil { 104 return err 105 } 106 if err := scm.AddOrReplacePartialRemote("origin", remote); err != nil { 107 return err 108 } 109 } 110 // We must specify a refspec here in order for patch to be able to set 111 // upstream to 'origin/main'. 112 if err := scm.Config("remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*"); err != nil { 113 return err 114 } 115 if cache != "" { 116 objPath := "objects" 117 if jirix.UsePartialClone(op.project.Remote) { 118 objPath = ".git/objects" 119 } 120 if err := os.WriteFile(filepath.Join(op.destination, ".git/objects/info/alternates"), []byte(filepath.Join(cache, objPath)+"\n"), 0644); err != nil { 121 return err 122 } 123 } 124 if err = fetchAll(jirix, op.project); err != nil { 125 return err 126 } 127 128 if cache != "" && jirix.Dissociate { 129 // Dissociating from the cache is slightly more complicated here, 130 // as `git fetch` does not have a `--dissociate` flag. As a result, 131 // we must invoke a dissociate manually. This involves running a 132 // repack, as well as removing the alternatives file. See the 133 // implementation of the dissociate flag in 134 // https://github.com/git/git/blob/main/builtin/clone.c#L1399 for 135 // more details. 136 opts := []gitutil.RepackOpt{gitutil.RepackAllOpt(true), gitutil.RemoveRedundantOpt(true)} 137 if err := gitutil.New(jirix).Repack(opts...); err != nil { 138 return err 139 } 140 if err := os.Remove(filepath.Join(op.destination, ".git/objects/info/alternates")); err != nil { 141 return err 142 } 143 } 144 } else { 145 r := remote 146 if cache != "" { 147 r = cache 148 defer func() { 149 if err := scm.AddOrReplaceRemote("origin", remote); err != nil { 150 jirix.Logger.Errorf("failed to set remote back to %v for project %+v", remote, op.project) 151 } 152 }() 153 } 154 opts := []gitutil.CloneOpt{gitutil.NoCheckoutOpt(true)} 155 if op.project.HistoryDepth > 0 { 156 opts = append(opts, gitutil.DepthOpt(op.project.HistoryDepth)) 157 } else { 158 // Shallow clones can not be used as as local git reference 159 opts = append(opts, gitutil.ReferenceOpt(cache)) 160 } 161 // Passing --filter=blob:none for a local clone is a no-op. 162 if (cache == r || cache == "") && jirix.UsePartialClone(op.project.Remote) { 163 opts = append(opts, gitutil.OmitBlobsOpt(true)) 164 } 165 if jirix.Dissociate { 166 opts = append(opts, gitutil.DissociateOpt(true)) 167 } 168 if err = clone(jirix, r, op.destination, opts...); err != nil { 169 return err 170 } 171 } 172 173 if err := os.Chmod(op.destination, os.FileMode(0755)); err != nil { 174 return fmtError(err) 175 } 176 177 if err := checkoutHeadRevision(jirix, op.project, false, false); err != nil { 178 return err 179 } 180 181 if err := writeMetadata(jirix, op.project, op.project.Path); err != nil { 182 return err 183 } 184 // Delete initial branch(es) 185 if branches, _, err := scm.GetBranches(); err != nil { 186 jirix.Logger.Warningf("not able to get branches for newly created project %s(%s)\n\n", op.project.Name, op.project.Path) 187 } else { 188 scm := gitutil.New(jirix, gitutil.RootDirOpt(op.project.Path)) 189 for _, b := range branches { 190 if err := scm.DeleteBranch(b); err != nil { 191 jirix.Logger.Warningf("not able to delete branch %s for project %s(%s)\n\n", b, op.project.Name, op.project.Path) 192 } 193 } 194 } 195 return nil 196 } 197 198 func (op createOperation) Run(jirix *jiri.X) (e error) { 199 path, perm := filepath.Dir(op.destination), os.FileMode(0755) 200 201 // Check the local file system. 202 if op.destination != jirix.Root { 203 if _, err := os.Stat(op.destination); err != nil { 204 if !os.IsNotExist(err) { 205 return fmtError(err) 206 } 207 } else { 208 if isEmpty, err := isEmpty(op.destination); err != nil { 209 return err 210 } else if !isEmpty { 211 return fmt.Errorf("cannot create %q as it already exists and is not empty", op.destination) 212 } else { 213 if err := os.RemoveAll(op.destination); err != nil { 214 return fmt.Errorf("Not able to delete %q", op.destination) 215 } 216 } 217 } 218 219 if err := os.MkdirAll(path, perm); err != nil { 220 return fmtError(err) 221 } 222 } 223 224 cache, err := op.project.CacheDirPath(jirix) 225 if err != nil { 226 return err 227 } 228 if !isPathDir(cache) { 229 cache = "" 230 } 231 232 if err := op.checkoutProject(jirix, cache); err != nil { 233 if op.destination != jirix.Root { 234 if err := os.RemoveAll(op.destination); err != nil { 235 jirix.Logger.Warningf("Not able to remove %q after create failed: %s", op.destination, err) 236 } 237 } 238 return err 239 } 240 241 // Remove branches for submodules if current project is a superproject. 242 if jirix.EnableSubmodules && op.project.GitSubmodules { 243 if err := removeAllSubmoduleBranches(jirix, op.project); err != nil { 244 return err 245 } 246 } 247 248 return nil 249 } 250 251 func (op createOperation) String() string { 252 return fmt.Sprintf("create project %q in %q and advance it to %q", op.project.Name, op.destination, fmtRevision(op.project.Revision)) 253 } 254 255 func (op createOperation) Test(jirix *jiri.X) error { 256 return nil 257 } 258 259 // deleteOperation represents the deletion of a project. 260 type deleteOperation struct { 261 commonOperation 262 } 263 264 func (op deleteOperation) Kind() string { 265 return deleteOpKind 266 } 267 268 func (op deleteOperation) Run(jirix *jiri.X) error { 269 if op.project.LocalConfig.Ignore { 270 jirix.Logger.Warningf("Project %s(%s) won't be deleted due to it's local-config\n\n", op.project.Name, op.source) 271 return nil 272 } 273 // Never delete projects with non-main branches, uncommitted work, or 274 // untracked content. 275 scm := gitutil.New(jirix, gitutil.RootDirOpt(op.project.Path)) 276 branches, _, err := scm.GetBranches() 277 if err != nil { 278 return fmt.Errorf("Cannot get branches for project %q: %s", op.Project().Name, err) 279 } 280 uncommitted, err := scm.HasUncommittedChanges() 281 if err != nil { 282 return fmt.Errorf("Cannot get uncommitted changes for project %q: %s", op.Project().Name, err) 283 } 284 untracked, err := scm.HasUntrackedFiles() 285 if err != nil { 286 return fmt.Errorf("Cannot get untracked changes for project %q: %s", op.Project().Name, err) 287 } 288 extraBranches := false 289 for _, branch := range branches { 290 if !strings.Contains(branch, "HEAD detached") { 291 extraBranches = true 292 break 293 } 294 } 295 296 if extraBranches || uncommitted || untracked { 297 gitDir, err := op.project.AbsoluteGitDir(jirix) 298 if err != nil { 299 return err 300 } 301 rmCommand := jirix.Color.Yellow("rm -rf %q", op.source) 302 unManageCommand := jirix.Color.Yellow("rm -rf %q", filepath.Join(gitDir, jiri.ProjectMetaDir)) 303 msg := "" 304 if extraBranches { 305 msg = fmt.Sprintf("Project %q won't be deleted as it contains branches", op.project.Name) 306 } else { 307 msg = fmt.Sprintf("Project %q won't be deleted as it might contain changes", op.project.Name) 308 } 309 msg += fmt.Sprintf("\nIf you no longer need it, invoke '%s'", rmCommand) 310 msg += fmt.Sprintf("\nIf you no longer want jiri to manage it, invoke '%s'\n\n", unManageCommand) 311 jirix.Logger.Warningf(msg) 312 return nil 313 } 314 315 if err := os.RemoveAll(op.source); err != nil { 316 return fmtError(err) 317 } 318 return removeEmptyParents(jirix, path.Dir(op.source)) 319 } 320 321 func removeEmptyParents(jirix *jiri.X, dir string) error { 322 isEmpty := func(name string) (bool, error) { 323 f, err := os.Open(name) 324 if err != nil { 325 return false, err 326 } 327 defer f.Close() 328 _, err = f.Readdirnames(1) 329 if err == io.EOF { 330 // empty dir 331 return true, nil 332 } 333 if err != nil { 334 return false, err 335 } 336 return false, nil 337 } 338 if !strings.HasPrefix(dir, jirix.Root) || jirix.Root == dir || dir == "" || dir == "." { 339 return nil 340 } 341 empty, err := isEmpty(dir) 342 if err != nil { 343 return err 344 } 345 if empty { 346 if err := os.Remove(dir); err != nil { 347 return err 348 } 349 jirix.Logger.Debugf("gc deleted empty parent directory: %v", dir) 350 return removeEmptyParents(jirix, path.Dir(dir)) 351 } 352 return nil 353 } 354 355 func (op deleteOperation) String() string { 356 return fmt.Sprintf("delete project %q from %q", op.project.Name, op.source) 357 } 358 359 func (op deleteOperation) Test(jirix *jiri.X) error { 360 if _, err := os.Stat(op.source); err != nil { 361 if os.IsNotExist(err) { 362 return fmt.Errorf("cannot delete %q as it does not exist", op.source) 363 } 364 return fmtError(err) 365 } 366 return nil 367 } 368 369 // moveOperation represents the relocation of a project. 370 type moveOperation struct { 371 commonOperation 372 rebaseTracked bool 373 rebaseUntracked bool 374 rebaseAll bool 375 rebaseSubmodules bool 376 snapshot bool 377 } 378 379 func (op moveOperation) Kind() string { 380 return moveOpKind 381 } 382 383 func (op moveOperation) Run(jirix *jiri.X) error { 384 if op.project.LocalConfig.Ignore { 385 jirix.Logger.Warningf("Project %s(%s) won't be moved or updated due to it's local-config\n\n", op.project.Name, op.source) 386 return nil 387 } 388 // If it was nested project it might have been moved with its parent project 389 if op.source != op.destination { 390 if err := renameDir(jirix, op.source, op.destination); err != nil { 391 return fmtError(err) 392 } 393 } 394 if err := syncProjectMaster(jirix, op.project, op.state, op.rebaseTracked, op.rebaseUntracked, op.rebaseAll, op.rebaseSubmodules, op.snapshot); err != nil { 395 return err 396 } 397 return writeMetadata(jirix, op.project, op.project.Path) 398 } 399 400 func (op moveOperation) String() string { 401 return fmt.Sprintf("move project %q located in %q to %q and advance it to %q", op.project.Name, op.source, op.destination, fmtRevision(op.project.Revision)) 402 } 403 404 func (op moveOperation) Test(jirix *jiri.X) error { 405 if _, err := os.Stat(op.source); err != nil { 406 if os.IsNotExist(err) { 407 return fmt.Errorf("cannot move %q to %q as the source does not exist", op.source, op.destination) 408 } 409 return fmtError(err) 410 } 411 if _, err := os.Stat(op.destination); err != nil { 412 if !os.IsNotExist(err) { 413 return fmtError(err) 414 } 415 } else { 416 // Check if the destination is our parent, and if we are the only child. 417 // This allows `jiri` to move repositories up a directory. 418 files, err := ioutil.ReadDir(op.destination) 419 if err != nil { 420 return fmtError(err) 421 } 422 if len(files) > 1 || (len(files) > 0 && filepath.Join(op.destination, files[0].Name()) != op.source) { 423 return fmt.Errorf("cannot move %q to %q as the destination already exists", op.source, op.destination) 424 } 425 } 426 return nil 427 } 428 429 // changeRemoteOperation represents the change of remote URL 430 type changeRemoteOperation struct { 431 commonOperation 432 rebaseTracked bool 433 rebaseUntracked bool 434 rebaseAll bool 435 rebaseSubmodules bool 436 snapshot bool 437 } 438 439 func (op changeRemoteOperation) Kind() string { 440 return changeRemoteOpKind 441 } 442 443 func (op changeRemoteOperation) Run(jirix *jiri.X) error { 444 if op.project.LocalConfig.Ignore || op.project.LocalConfig.NoUpdate { 445 jirix.Logger.Warningf("Project %s(%s) won't be updated due to it's local-config. It has a changed remote\n\n", op.project.Name, op.project.Path) 446 return nil 447 } 448 git := gitutil.New(jirix, gitutil.RootDirOpt(op.project.Path)) 449 tempRemote := "new-remote-origin" 450 if err := git.AddRemote(tempRemote, op.project.Remote); err != nil { 451 return err 452 } 453 defer git.DeleteRemote(tempRemote) 454 455 if err := fetch(jirix, op.project.Path, tempRemote); err != nil { 456 return err 457 } 458 459 // Check for all leaf commits in new remote 460 for _, branch := range op.state.Branches { 461 if containingBranches, err := git.GetRemoteBranchesContaining(branch.Revision); err != nil { 462 return err 463 } else { 464 foundBranch := false 465 for _, remoteBranchName := range containingBranches { 466 if strings.HasPrefix(remoteBranchName, tempRemote) { 467 foundBranch = true 468 break 469 } 470 } 471 if !foundBranch { 472 jirix.Logger.Errorf("Note: For project %q(%v), remote url has changed. Its branch %q is on a commit", op.project.Name, op.project.Path, branch.Name) 473 jirix.Logger.Errorf("which is not in new remote(%v). Please manually reset your branches or move", op.project.Remote) 474 jirix.Logger.Errorf("your project folder out of the root and try again") 475 return nil 476 } 477 478 } 479 } 480 481 // Everything ok, change the remote url 482 if err := git.SetRemoteUrl("origin", op.project.Remote); err != nil { 483 return err 484 } 485 486 if err := fetch(jirix, op.project.Path, "", gitutil.AllOpt(true), gitutil.PruneOpt(true)); err != nil { 487 return err 488 } 489 490 if err := syncProjectMaster(jirix, op.project, op.state, op.rebaseTracked, op.rebaseUntracked, op.rebaseAll, op.rebaseSubmodules, op.snapshot); err != nil { 491 return err 492 } 493 494 return writeMetadata(jirix, op.project, op.project.Path) 495 } 496 497 func (op changeRemoteOperation) String() string { 498 return fmt.Sprintf("Change remote of project %q to %q and update it to %q", op.project.Name, op.project.Remote, fmtRevision(op.project.Revision)) 499 } 500 501 func (op changeRemoteOperation) Test(jirix *jiri.X) error { 502 return nil 503 } 504 505 // updateOperation represents the update of a project. 506 type updateOperation struct { 507 commonOperation 508 rebaseTracked bool 509 rebaseUntracked bool 510 rebaseAll bool 511 rebaseSubmodules bool 512 snapshot bool 513 } 514 515 func (op updateOperation) Kind() string { 516 return updateOpKind 517 } 518 519 func (op updateOperation) Run(jirix *jiri.X) error { 520 if err := syncProjectMaster(jirix, op.project, op.state, op.rebaseTracked, op.rebaseUntracked, op.rebaseAll, op.rebaseSubmodules, op.snapshot); err != nil { 521 return err 522 } 523 // If we enabled submodules and current project is a superproject, we need to remove intial branches and foo branch. 524 if jirix.EnableSubmodules && op.project.GitSubmodules { 525 if err := removeSubmoduleBranches(jirix, op.project, SubmoduleLocalFlagBranch); err != nil { 526 return err 527 } 528 } 529 return writeMetadata(jirix, op.project, op.project.Path) 530 } 531 532 func (op updateOperation) String() string { 533 return fmt.Sprintf("advance/rebase project %q located in %q to %q", op.project.Name, op.source, fmtRevision(op.project.Revision)) 534 } 535 536 func (op updateOperation) Test(jirix *jiri.X) error { 537 return nil 538 } 539 540 // nullOperation represents a noop. It is used for logging and adding project 541 // information to the current manifest. 542 type nullOperation struct { 543 commonOperation 544 } 545 546 func (op nullOperation) Kind() string { 547 return nullOpKind 548 } 549 550 func (op nullOperation) Run(jirix *jiri.X) error { 551 return writeMetadata(jirix, op.project, op.project.Path) 552 } 553 554 func (op nullOperation) String() string { 555 return fmt.Sprintf("project %q located in %q at revision %q is up-to-date", op.project.Name, op.source, fmtRevision(op.project.Revision)) 556 } 557 558 func (op nullOperation) Test(jirix *jiri.X) error { 559 return nil 560 } 561 562 // operations is a sortable collection of operations 563 type operations []operation 564 565 // Len returns the length of the collection. 566 func (ops operations) Len() int { 567 return len(ops) 568 } 569 570 // Less defines the order of operations. Operations are ordered first 571 // by their type and then by their project path. 572 // 573 // The order in which operation types are defined determines the order 574 // in which operations are performed. For correctness and also to 575 // minimize the chance of a conflict, the delete operations should 576 // happen before change-remote operations, which should happen before move 577 // operations. If two create operations make nested directories, the 578 // outermost should be created first. 579 // 580 // When 2 operations have a parent/child relationship, we attempt to do the 581 // following: 582 // 1) If the child is moving further down the directory tree, we order it 583 // before the parent's update with the assumption the parent may expand into 584 // the child's current directory. 585 // 2) If the child is moving up the directory tree, we order it after the 586 // parent's update with the assumption the parent may be contracting to make 587 // space for the child. 588 // 3) If the child is being created, we follow the same logic as #2. 589 // 4) We sub order all the moves from outward moves to inward moves so the 590 // logic of #1 and #2 function as expected within the sort. 591 func (ops operations) Less(i, j int) bool { 592 isSubdir := func(child, parent string) bool { 593 return strings.HasPrefix(child, parent+string(filepath.Separator)) 594 } 595 596 opKindToPriority := func(kind string) int { 597 var priortity int 598 switch kind { 599 case deleteOpKind: 600 priortity = 0 601 case changeRemoteOpKind: 602 priortity = 1 603 case moveOpKind: 604 priortity = 2 605 case updateOpKind: 606 priortity = 3 607 case createOpKind: 608 priortity = 4 609 case nullOpKind: 610 priortity = 5 611 } 612 return priortity 613 } 614 615 if ops[i].Kind() == moveOpKind { 616 if ops[j].Kind() == updateOpKind { 617 // Move is in a child project of Update 618 if isSubdir(ops[i].Source(), ops[j].Source()) { 619 // Move out 620 if isSubdir(ops[i].Source(), ops[i].Destination()) { 621 return false // Move happens after update 622 } 623 } 624 } 625 if ops[j].Kind() == createOpKind { 626 // Create is the parent of the move destination 627 if isSubdir(ops[i].Destination(), ops[j].Destination()) { 628 return false // Move happens after create 629 } 630 } 631 if ops[j].Kind() == moveOpKind { 632 // Move out 633 if isSubdir(ops[i].Destination(), ops[i].Source()) { 634 return true 635 // Move in 636 } else if isSubdir(ops[i].Source(), ops[i].Destination()) { 637 return false 638 } 639 } 640 } 641 642 if ops[i].Kind() == createOpKind { 643 if ops[j].Kind() == moveOpKind { 644 // Move out 645 if isSubdir(ops[j].Destination(), ops[i].Destination()) { 646 return true 647 } 648 } 649 if ops[j].Kind() == updateOpKind { 650 // Create in child 651 if isSubdir(ops[i].Destination(), ops[j].Destination()) { 652 return false 653 } 654 } 655 } 656 657 if ops[i].Kind() == updateOpKind { 658 if ops[j].Kind() == moveOpKind || ops[j].Kind() == createOpKind { 659 // Op in child 660 if isSubdir(ops[j].Destination(), ops[i].Source()) { 661 // Move out 662 if ops[j].Kind() == moveOpKind && isSubdir(ops[j].Destination(), ops[j].Source()) { 663 return false // Move out happens before update 664 } 665 return true 666 } 667 } 668 } 669 670 if ops[i].Kind() != ops[j].Kind() { 671 return opKindToPriority(ops[i].Kind()) < opKindToPriority(ops[j].Kind()) 672 } 673 674 if ops[i].Kind() == deleteOpKind { 675 return ops[i].Source() > ops[j].Source() 676 } 677 678 return ops[i].Destination() < ops[j].Destination() 679 } 680 681 // Swap swaps two elements of the collection. 682 func (ops operations) Swap(i, j int) { 683 ops[i], ops[j] = ops[j], ops[i] 684 } 685 686 // computeOperations inputs a set of projects to update and the set of 687 // current and new projects (as defined by contents of the local file 688 // system and manifest file respectively) and outputs a collection of 689 // operations that describe the actions needed to update the target 690 // projects. 691 // In the case of submodules, computeOperation will check for necessary 692 // deletions of jiri projects and initialize submodules in place of projects. 693 func computeOperations(jirix *jiri.X, localProjects, remoteProjects Projects, states map[ProjectKey]*ProjectState, rebaseTracked, rebaseUntracked, rebaseAll, rebaseSubmodules, snapshot bool) operations { 694 result := operations{} 695 allProjects := map[ProjectKey]bool{} 696 for _, p := range localProjects { 697 allProjects[p.Key()] = true 698 } 699 for _, p := range remoteProjects { 700 allProjects[p.Key()] = true 701 } 702 // When we are switching submodules to projects, we deinit all of the current existing local submodules. 703 if !jirix.EnableSubmodules && containLocalSubmodules(localProjects) { 704 scm := gitutil.New(jirix, gitutil.RootDirOpt(jirix.Root)) 705 scm.SubmoduleDeinit() 706 } 707 for key := range allProjects { 708 var local, remote *Project 709 var state *ProjectState 710 if project, ok := localProjects[key]; ok { 711 local = &project 712 } 713 if project, ok := remoteProjects[key]; ok { 714 // update remote local config 715 if local != nil { 716 project.LocalConfig = local.LocalConfig 717 remoteProjects[key] = project 718 } 719 remote = &project 720 } 721 if s, ok := states[key]; ok { 722 state = s 723 } 724 result = append(result, computeOp(jirix, local, remote, state, rebaseTracked, rebaseUntracked, rebaseAll, rebaseSubmodules, snapshot)) 725 } 726 sort.Sort(result) 727 return result 728 } 729 730 func computeOp(jirix *jiri.X, local, remote *Project, state *ProjectState, rebaseTracked, rebaseUntracked, rebaseAll, rebaseSubmodules, snapshot bool) operation { 731 switch { 732 case local == nil && remote != nil: 733 return createOperation{commonOperation{ 734 destination: remote.Path, 735 project: *remote, 736 source: "", 737 }} 738 case local != nil && remote == nil: 739 // When submoduels are enabled, all submodules are removed from remote projects, so submodules from remote are nil. 740 // We skip operations on submodules when we enabled submodules and rely on superproject updates. 741 if jirix.EnableSubmodules && local.IsSubmodule { 742 return nullOperation{commonOperation{ 743 project: *local, 744 source: local.Path, 745 state: *state, 746 }} 747 } 748 return deleteOperation{commonOperation{ 749 destination: "", 750 project: *local, 751 source: local.Path, 752 }} 753 case local != nil && remote != nil: 754 // When we are switching from submodules to projects, submodules are all removed and all projects need to be created new. 755 if !jirix.EnableSubmodules && local.IsSubmodule { 756 return createOperation{commonOperation{ 757 destination: remote.Path, 758 project: *remote, 759 source: "", 760 }} 761 } 762 763 localBranchesNeedUpdating := false 764 if !snapshot { 765 cb := state.CurrentBranch 766 if rebaseAll { 767 for _, branch := range state.Branches { 768 if branch.Tracking != nil { 769 if branch.Revision != branch.Tracking.Revision { 770 localBranchesNeedUpdating = true 771 break 772 } 773 } else if rebaseUntracked && rebaseAll { 774 // We put checks for untracked-branch updating in syncProjectMaster function 775 localBranchesNeedUpdating = true 776 break 777 } 778 } 779 } else if cb.Name != "" && cb.Tracking != nil && cb.Revision != cb.Tracking.Revision { 780 localBranchesNeedUpdating = true 781 } 782 } 783 switch { 784 case local.Remote != remote.Remote: 785 return changeRemoteOperation{commonOperation{ 786 destination: remote.Path, 787 project: *remote, 788 source: local.Path, 789 state: *state, 790 }, rebaseTracked, rebaseUntracked, rebaseAll, rebaseSubmodules, snapshot} 791 case local.Path != remote.Path: 792 if remote.Path == jirix.Root { 793 return createOperation{commonOperation{ 794 destination: remote.Path, 795 project: *remote, 796 source: "", 797 }} 798 } 799 // moveOperation also does an update, so we don't need to check the 800 // revision here. 801 return moveOperation{commonOperation{ 802 destination: remote.Path, 803 project: *remote, 804 source: local.Path, 805 state: *state, 806 }, rebaseTracked, rebaseUntracked, rebaseAll, rebaseSubmodules, snapshot} 807 // No need to update projects when current project exists as a submodule 808 case jirix.EnableSubmodules && local.IsSubmodule: 809 return nullOperation{commonOperation{ 810 destination: remote.Path, 811 project: *remote, 812 source: local.Path, 813 state: *state, 814 }} 815 case snapshot && local.Revision != remote.Revision: 816 return updateOperation{commonOperation{ 817 destination: remote.Path, 818 project: *remote, 819 source: local.Path, 820 state: *state, 821 }, rebaseTracked, rebaseUntracked, rebaseAll, rebaseSubmodules, snapshot} 822 case jirix.EnableSubmodules && local.GitSubmodules: 823 // Always update superproject when submoduels are enabled. 824 return updateOperation{commonOperation{ 825 destination: remote.Path, 826 project: *remote, 827 source: local.Path, 828 state: *state, 829 }, rebaseTracked, rebaseUntracked, rebaseAll, rebaseSubmodules, snapshot} 830 case localBranchesNeedUpdating || (state.CurrentBranch.Name == "" && local.Revision != remote.Revision): 831 return updateOperation{commonOperation{ 832 destination: remote.Path, 833 project: *remote, 834 source: local.Path, 835 state: *state, 836 }, rebaseTracked, rebaseUntracked, rebaseAll, rebaseSubmodules, snapshot} 837 case state.CurrentBranch.Tracking == nil && local.Revision != remote.Revision: 838 return updateOperation{commonOperation{ 839 destination: remote.Path, 840 project: *remote, 841 source: local.Path, 842 state: *state, 843 }, rebaseTracked, rebaseUntracked, rebaseAll, rebaseSubmodules, snapshot} 844 default: 845 return nullOperation{commonOperation{ 846 destination: remote.Path, 847 project: *remote, 848 source: local.Path, 849 state: *state, 850 }} 851 } 852 default: 853 panic("jiri: computeOp called with nil local and remote") 854 } 855 } 856 857 // This function creates worktree and runs create operation in parallel 858 func runCreateOperations(jirix *jiri.X, ops []createOperation) MultiError { 859 jirix.TimerPush("create operations") 860 defer jirix.TimerPop() 861 count := len(ops) 862 if count == 0 { 863 return nil 864 } 865 866 type workTree struct { 867 // dir is the top level directory in which operations will be performed 868 dir string 869 // op is an ordered list of operations that must be performed serially, 870 // affecting dir 871 ops []operation 872 // after contains a tree of work that must be performed after ops 873 after map[string]*workTree 874 } 875 head := &workTree{ 876 dir: "", 877 ops: []operation{}, 878 after: make(map[string]*workTree), 879 } 880 881 for _, op := range ops { 882 883 node := head 884 parts := strings.Split(op.Project().Path, string(filepath.Separator)) 885 // walk down the file path tree, creating any work tree nodes as required 886 for _, part := range parts { 887 if part == "" { 888 continue 889 } 890 next, ok := node.after[part] 891 if !ok { 892 next = &workTree{ 893 dir: part, 894 ops: []operation{}, 895 after: make(map[string]*workTree), 896 } 897 node.after[part] = next 898 } 899 node = next 900 } 901 node.ops = append(node.ops, op) 902 } 903 904 workQueue := make(chan *workTree, count) 905 errs := make(chan error, count) 906 var wg sync.WaitGroup 907 processTree := func(tree *workTree) { 908 defer wg.Done() 909 for _, op := range tree.ops { 910 logMsg := fmt.Sprintf("Creating project %q", op.Project().Name) 911 task := jirix.Logger.AddTaskMsg(logMsg) 912 jirix.Logger.Debugf("%v", op) 913 if err := op.Run(jirix); err != nil { 914 task.Done() 915 errs <- fmt.Errorf("%s: %s", logMsg, err) 916 return 917 } 918 task.Done() 919 } 920 for _, v := range tree.after { 921 wg.Add(1) 922 workQueue <- v 923 } 924 } 925 wg.Add(1) 926 workQueue <- head 927 for i := uint(0); i < jirix.Jobs; i++ { 928 go func() { 929 for tree := range workQueue { 930 processTree(tree) 931 } 932 }() 933 } 934 wg.Wait() 935 close(workQueue) 936 close(errs) 937 938 var multiErr MultiError 939 for err := range errs { 940 multiErr = append(multiErr, err) 941 } 942 return multiErr 943 } 944 945 type PathTrie struct { 946 current string 947 children map[string]*PathTrie 948 } 949 950 func NewPathTrie() *PathTrie { 951 return &PathTrie{ 952 current: "", 953 children: make(map[string]*PathTrie), 954 } 955 } 956 957 func (p *PathTrie) Contains(path string) bool { 958 parts := strings.Split(path, string(filepath.Separator)) 959 node := p 960 for _, part := range parts { 961 if part == "" { 962 continue 963 } 964 child, ok := node.children[part] 965 if !ok { 966 return false 967 } 968 node = child 969 } 970 return true 971 } 972 973 func (p *PathTrie) Insert(path string) { 974 parts := strings.Split(path, string(filepath.Separator)) 975 node := p 976 for _, part := range parts { 977 if part == "" { 978 continue 979 } 980 child, ok := node.children[part] 981 if !ok { 982 child = &PathTrie{ 983 current: part, 984 children: make(map[string]*PathTrie), 985 } 986 node.children[part] = child 987 } 988 node = child 989 } 990 } 991 992 func runDeleteOperations(jirix *jiri.X, ops []deleteOperation, gc bool) error { 993 jirix.TimerPush("delete operations") 994 defer jirix.TimerPop() 995 if len(ops) == 0 { 996 return nil 997 } 998 notDeleted := NewPathTrie() 999 if !gc { 1000 msg := fmt.Sprintf("%d project(s) is/are marked to be deleted. Run '%s' to delete them.", len(ops), jirix.Color.Yellow("jiri update -gc")) 1001 if jirix.Logger.LoggerLevel < log.DebugLevel { 1002 msg = fmt.Sprintf("%s\nOr run '%s' or '%s' to see the list of projects.", msg, jirix.Color.Yellow("jiri update -v"), jirix.Color.Yellow("jiri status -d")) 1003 } 1004 jirix.Logger.Warningf("%s\n\n", msg) 1005 if jirix.Logger.LoggerLevel >= log.DebugLevel { 1006 msg = "List of project(s) marked to be deleted:" 1007 for _, op := range ops { 1008 msg = fmt.Sprintf("%s\nName: %s, Path: '%s'", msg, jirix.Color.Yellow(op.project.Name), jirix.Color.Yellow(op.source)) 1009 } 1010 jirix.Logger.Debugf("%s\n\n", msg) 1011 } 1012 return nil 1013 } 1014 for _, op := range ops { 1015 if notDeleted.Contains(op.Project().Path) { 1016 // not deleting project, add it to trie 1017 notDeleted.Insert(op.source) 1018 rmCommand := jirix.Color.Yellow("rm -rf %q", op.source) 1019 msg := fmt.Sprintf("Project %q won't be deleted because of its sub project(s)", op.project.Name) 1020 msg += fmt.Sprintf("\nIf you no longer need it, invoke '%s'\n\n", rmCommand) 1021 jirix.Logger.Warningf(msg) 1022 continue 1023 } 1024 logMsg := fmt.Sprintf("Deleting project %q", op.Project().Name) 1025 task := jirix.Logger.AddTaskMsg(logMsg) 1026 jirix.Logger.Debugf("%s", op) 1027 if err := op.Run(jirix); err != nil { 1028 task.Done() 1029 return fmt.Errorf("%s: %s", logMsg, err) 1030 } 1031 task.Done() 1032 if _, err := os.Stat(op.source); err == nil { 1033 // project not deleted, add it to trie 1034 notDeleted.Insert(op.source) 1035 } else if err != nil && !os.IsNotExist(err) { 1036 return fmt.Errorf("Checking if %q exists", op.source) 1037 } 1038 } 1039 return nil 1040 } 1041 1042 func runMoveOperations(jirix *jiri.X, ops []moveOperation) error { 1043 jirix.TimerPush("move operations") 1044 defer jirix.TimerPop() 1045 parentSrcPath := "" 1046 parentDestPath := "" 1047 for _, op := range ops { 1048 if parentSrcPath != "" && strings.HasPrefix(op.source, parentSrcPath) { 1049 op.source = filepath.Join(parentDestPath, strings.Replace(op.source, parentSrcPath, "", 1)) 1050 } else { 1051 parentSrcPath = op.source 1052 parentDestPath = op.destination 1053 } 1054 logMsg := fmt.Sprintf("Moving and updating project %q", op.Project().Name) 1055 task := jirix.Logger.AddTaskMsg(logMsg) 1056 jirix.Logger.Debugf("%s", op) 1057 if err := op.Run(jirix); err != nil { 1058 task.Done() 1059 return fmt.Errorf("%s: %s", logMsg, err) 1060 } 1061 task.Done() 1062 } 1063 return nil 1064 } 1065 1066 func runCommonOperations(jirix *jiri.X, ops operations, loglevel log.LogLevel) error { 1067 jirix.TimerPush("common operations") 1068 defer jirix.TimerPop() 1069 for _, op := range ops { 1070 logMsg := fmt.Sprintf("Updating project %q", op.Project().Name) 1071 task := jirix.Logger.AddTaskMsg(logMsg) 1072 jirix.Logger.Logf(loglevel, "%s", op) 1073 if err := op.Run(jirix); err != nil { 1074 task.Done() 1075 return fmt.Errorf("%s: %s", logMsg, err) 1076 } 1077 task.Done() 1078 } 1079 return nil 1080 } 1081 1082 func renameDir(jirix *jiri.X, src, dst string) error { 1083 // Parent directory permissions 1084 perm := os.FileMode(0755) 1085 swapDir := jirix.SwapDir() 1086 1087 // Hash src path as swap dir name 1088 h := fnv.New32a() 1089 h.Write([]byte(src)) 1090 tmp := filepath.Join(swapDir, fmt.Sprintf("%d", h.Sum32())) 1091 // Ensure .jiri_root/swap exists 1092 if err := os.MkdirAll(swapDir, perm); err != nil { 1093 return err 1094 } 1095 1096 // Move src -> tmp 1097 if err := osutil.Rename(src, tmp); err != nil { 1098 return err 1099 } 1100 1101 if err := removeEmptyParents(jirix, dst); err != nil { 1102 jirix.Logger.Tracef("Could not remove empty directories for %s", dst) 1103 } 1104 1105 // Ensure the dst's parent exists, it may have 1106 // been within src 1107 parentDir := filepath.Dir(dst) 1108 if err := os.MkdirAll(parentDir, perm); err != nil { 1109 return err 1110 } 1111 1112 // Move tmp -> dst 1113 if err := osutil.Rename(tmp, dst); err != nil { 1114 if err := osutil.Rename(tmp, src); err != nil { 1115 jirix.Logger.Errorf("Could not move %s to %s, original contents are in %s. Please complete the move manually", src, dst, tmp) 1116 } 1117 return err 1118 } 1119 return nil 1120 }