github.com/btwiuse/jiri@v0.0.0-20191125065820-53353bcfef54/project/manifest.go (about) 1 // Copyright 2017 The Fuchsia Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package project 6 7 import ( 8 "bytes" 9 "context" 10 "encoding/json" 11 "encoding/xml" 12 "errors" 13 "fmt" 14 "hash/fnv" 15 "io" 16 "io/ioutil" 17 "net/url" 18 "os" 19 "os/exec" 20 "path" 21 "path/filepath" 22 "regexp" 23 "sort" 24 "strings" 25 "text/template" 26 "time" 27 28 "github.com/btwiuse/jiri" 29 "github.com/btwiuse/jiri/cipd" 30 "github.com/btwiuse/jiri/envvar" 31 "github.com/btwiuse/jiri/gerrit" 32 "github.com/btwiuse/jiri/gitutil" 33 "github.com/btwiuse/jiri/retry" 34 "golang.org/x/net/publicsuffix" 35 ) 36 37 // Manifest represents a setting used for updating the universe. 38 type Manifest struct { 39 Version string `xml:"version,attr,omitempty"` 40 Attributes string `xml:"attributes,attr,omitempty"` 41 Imports []Import `xml:"imports>import"` 42 LocalImports []LocalImport `xml:"imports>localimport"` 43 Projects []Project `xml:"projects>project"` 44 ProjectOverrides []Project `xml:"overrides>project"` 45 ImportOverrides []Import `xml:"overrides>import"` 46 Hooks []Hook `xml:"hooks>hook"` 47 Packages []Package `xml:"packages>package"` 48 XMLName struct{} `xml:"manifest"` 49 } 50 51 // ManifestFromBytes returns a manifest parsed from data, with defaults filled 52 // in. 53 func ManifestFromBytes(data []byte) (*Manifest, error) { 54 m := new(Manifest) 55 if len(data) > 0 { 56 if err := xml.Unmarshal(data, m); err != nil { 57 return nil, err 58 } 59 } 60 if err := m.fillDefaults(); err != nil { 61 return nil, err 62 } 63 return m, nil 64 } 65 66 // ManifestFromFile returns a manifest parsed from the contents of filename, 67 // with defaults filled in. 68 // 69 // Note that unlike ProjectFromFile, ManifestFromFile does not convert project 70 // paths to absolute paths because it's possible to load a manifest with a 71 // specific root directory different from jirix.Root. The usual way to load a 72 // manifest is through LoadManifest, which does absolutize the paths, and uses 73 // the correct root directory. 74 func ManifestFromFile(jirix *jiri.X, filename string) (*Manifest, error) { 75 data, err := ioutil.ReadFile(filename) 76 if err != nil { 77 return nil, fmtError(err) 78 } 79 m, err := ManifestFromBytes(data) 80 if err != nil { 81 return nil, fmt.Errorf("invalid manifest %s: %v", filename, err) 82 } 83 return m, nil 84 } 85 86 var ( 87 newlineBytes = []byte("\n") 88 emptyImportsBytes = []byte("\n <imports></imports>\n") 89 emptyProjectsBytes = []byte("\n <projects></projects>\n") 90 emptyOverridesBytes = []byte("\n <overrides></overrides>\n") 91 emptyHooksBytes = []byte("\n <hooks></hooks>\n") 92 emptyPackagesBytes = []byte("\n <packages></packages>\n") 93 94 endElemBytes = []byte("/>\n") 95 endImportBytes = []byte("></import>\n") 96 endLocalImportBytes = []byte("></localimport>\n") 97 endProjectBytes = []byte("></project>\n") 98 endHookBytes = []byte("></hook>\n") 99 endPackageBytes = []byte("></package>\n") 100 101 endImportSoloBytes = []byte("></import>") 102 endProjectSoloBytes = []byte("></project>") 103 endElemSoloBytes = []byte("/>") 104 105 errGitHookNotRequired = errors.New("git hooks are not required") 106 ) 107 108 const ( 109 fuchsiaGerritHost = "https://fuchsia-review.googlesource.com" 110 ) 111 112 // deepCopy returns a deep copy of Manifest. 113 func (m *Manifest) deepCopy() *Manifest { 114 x := new(Manifest) 115 x.Imports = append([]Import(nil), m.Imports...) 116 x.LocalImports = append([]LocalImport(nil), m.LocalImports...) 117 x.Projects = append([]Project(nil), m.Projects...) 118 x.ProjectOverrides = append([]Project(nil), m.ProjectOverrides...) 119 x.ImportOverrides = append([]Import(nil), m.ImportOverrides...) 120 x.Hooks = append([]Hook(nil), m.Hooks...) 121 x.Packages = append([]Package(nil), m.Packages...) 122 x.Version = m.Version 123 x.Attributes = m.Attributes 124 return x 125 } 126 127 // ToBytes returns m as serialized bytes, with defaults unfilled. 128 func (m *Manifest) ToBytes() ([]byte, error) { 129 m = m.deepCopy() // avoid changing manifest when unfilling defaults. 130 if err := m.unfillDefaults(); err != nil { 131 return nil, err 132 } 133 data, err := xml.MarshalIndent(m, "", " ") 134 if err != nil { 135 return nil, fmt.Errorf("manifest xml.Marshal failed: %v", err) 136 } 137 // It's hard (impossible?) to get xml.Marshal to elide some of the empty 138 // elements, or produce short empty elements, so we post-process the data. 139 data = bytes.Replace(data, emptyImportsBytes, newlineBytes, -1) 140 data = bytes.Replace(data, emptyProjectsBytes, newlineBytes, -1) 141 data = bytes.Replace(data, emptyOverridesBytes, newlineBytes, -1) 142 data = bytes.Replace(data, emptyHooksBytes, newlineBytes, -1) 143 data = bytes.Replace(data, emptyPackagesBytes, newlineBytes, -1) 144 data = bytes.Replace(data, endImportBytes, endElemBytes, -1) 145 data = bytes.Replace(data, endLocalImportBytes, endElemBytes, -1) 146 data = bytes.Replace(data, endProjectBytes, endElemBytes, -1) 147 data = bytes.Replace(data, endHookBytes, endElemBytes, -1) 148 data = bytes.Replace(data, endPackageBytes, endElemBytes, -1) 149 if !bytes.HasSuffix(data, newlineBytes) { 150 data = append(data, '\n') 151 } 152 return data, nil 153 } 154 155 // ToFile writes the manifest m to a file with the given filename, with 156 // defaults unfilled and all project paths relative to the jiri root. 157 func (m *Manifest) ToFile(jirix *jiri.X, filename string) error { 158 // Replace absolute paths with relative paths to make it possible to move 159 // the root directory locally. 160 projects := []Project{} 161 for _, project := range m.Projects { 162 if err := project.relativizePaths(jirix.Root); err != nil { 163 return err 164 } 165 projects = append(projects, project) 166 } 167 // Sort the projects and hooks to ensure that the output of "jiri 168 // snapshot" is deterministic. Sorting the hooks by name allows 169 // some control over the ordering of the hooks in case that is 170 // necessary. 171 sort.Sort(ProjectsByPath(projects)) 172 m.Projects = projects 173 sort.Sort(HooksByName(m.Hooks)) 174 data, err := m.ToBytes() 175 if err != nil { 176 return err 177 } 178 return safeWriteFile(jirix, filename, data) 179 } 180 181 func (m *Manifest) fillDefaults() error { 182 for index := range m.Imports { 183 if err := m.Imports[index].fillDefaults(); err != nil { 184 return err 185 } 186 } 187 for index := range m.LocalImports { 188 if err := m.LocalImports[index].validate(); err != nil { 189 return err 190 } 191 } 192 for index := range m.Projects { 193 if err := m.Projects[index].fillDefaults(); err != nil { 194 return err 195 } 196 } 197 for index := range m.ProjectOverrides { 198 if err := m.ProjectOverrides[index].fillDefaults(); err != nil { 199 return err 200 } 201 } 202 for index := range m.ImportOverrides { 203 if err := m.ImportOverrides[index].fillDefaults(); err != nil { 204 return err 205 } 206 } 207 return nil 208 } 209 210 func (m *Manifest) unfillDefaults() error { 211 for index := range m.Imports { 212 if err := m.Imports[index].unfillDefaults(); err != nil { 213 return err 214 } 215 } 216 for index := range m.LocalImports { 217 if err := m.LocalImports[index].validate(); err != nil { 218 return err 219 } 220 } 221 for index := range m.Projects { 222 if err := m.Projects[index].unfillDefaults(); err != nil { 223 return err 224 } 225 } 226 for index := range m.ProjectOverrides { 227 if err := m.ProjectOverrides[index].unfillDefaults(); err != nil { 228 return err 229 } 230 } 231 for index := range m.ImportOverrides { 232 if err := m.ImportOverrides[index].unfillDefaults(); err != nil { 233 return err 234 } 235 } 236 return nil 237 } 238 239 // Import represents a remote manifest import. 240 type Import struct { 241 // Manifest file to use from the remote manifest project. 242 Manifest string `xml:"manifest,attr,omitempty"` 243 // Name is the name of the remote manifest project, used to determine the 244 // project key. 245 Name string `xml:"name,attr,omitempty"` 246 // Remote is the remote manifest project to import. 247 Remote string `xml:"remote,attr,omitempty"` 248 // Revision is the revison to checkout, 249 // this takes precedence over RemoteBranch 250 Revision string `xml:"revision,attr,omitempty"` 251 // RemoteBranch is the name of the remote branch to track. 252 RemoteBranch string `xml:"remotebranch,attr,omitempty"` 253 // Root path, prepended to all project paths specified in the manifest file. 254 Root string `xml:"root,attr,omitempty"` 255 XMLName struct{} `xml:"import"` 256 } 257 258 func (i *Import) fillDefaults() error { 259 if i.RemoteBranch == "" { 260 i.RemoteBranch = "master" 261 } 262 if i.Revision == "" { 263 i.Revision = "HEAD" 264 } 265 return i.validate() 266 } 267 268 func (i *Import) RemoveDefaults() { 269 if i.RemoteBranch == "master" { 270 i.RemoteBranch = "" 271 } 272 if i.Revision == "HEAD" { 273 i.Revision = "" 274 } 275 } 276 277 func (i *Import) unfillDefaults() error { 278 i.RemoveDefaults() 279 return i.validate() 280 } 281 282 func (i *Import) validate() error { 283 if i.Manifest == "" || i.Remote == "" { 284 return fmt.Errorf("bad import: both manifest and remote must be specified") 285 } 286 return nil 287 } 288 289 func (i *Import) toProject(path string) (Project, error) { 290 p := Project{ 291 Name: i.Name, 292 Path: path, 293 Remote: i.Remote, 294 Revision: i.Revision, 295 RemoteBranch: i.RemoteBranch, 296 } 297 err := p.fillDefaults() 298 return p, err 299 } 300 301 // ProjectKey returns the unique ProjectKey for the imported project. 302 func (i *Import) ProjectKey() ProjectKey { 303 return MakeProjectKey(i.Name, i.Remote) 304 } 305 306 // projectKeyFileName returns a file name based on the ProjectKey. 307 func (i *Import) projectKeyFileName() string { 308 // TODO(toddw): Disallow weird characters from project names. 309 hash := fnv.New64a() 310 hash.Write([]byte(i.ProjectKey())) 311 return fmt.Sprintf("%s_%x", i.Name, hash.Sum64()) 312 } 313 314 // cycleKey returns a key based on the remote and manifest, used for 315 // cycle-detection. It's only valid for new-style remote imports; it's empty 316 // for the old-style local imports. 317 func (i *Import) cycleKey() string { 318 if i.Remote == "" { 319 return "" 320 } 321 // We don't join the remote and manifest with a slash or any other url-safe 322 // character, since that might not be unique. E.g. 323 // remote: https://foo.com/a/b remote: https://foo.com/a 324 // manifest: c manifest: b/c 325 // In both cases, the key would be https://foo.com/a/b/c. 326 return i.Remote + " + " + i.Manifest 327 } 328 329 func (i *Import) update(o *Import) { 330 if o.Manifest != "" { 331 i.Manifest = o.Manifest 332 } 333 if o.Name != "" { 334 i.Name = o.Name 335 } 336 if o.Remote != "" { 337 i.Remote = o.Remote 338 } 339 if o.Revision != "" { 340 i.Revision = o.Revision 341 } 342 if o.RemoteBranch != "" { 343 i.RemoteBranch = o.RemoteBranch 344 } 345 if o.Root != "" { 346 i.Root = o.Root 347 } 348 } 349 350 // LocalImport represents a local manifest import. 351 type LocalImport struct { 352 // Manifest file to import from. 353 File string `xml:"file,attr,omitempty"` 354 XMLName struct{} `xml:"localimport"` 355 } 356 357 func (i *LocalImport) validate() error { 358 if i.File == "" { 359 return fmt.Errorf("bad localimport: must specify file: %+v", *i) 360 } 361 return nil 362 } 363 364 type LocalConfig struct { 365 Ignore bool `xml:"ignore"` 366 NoUpdate bool `xml:"no-update"` 367 NoRebase bool `xml:"no-rebase"` 368 XMLName struct{} `xml:"config"` 369 } 370 371 // Reads localConfig from given reader. Returns incorrect bytes 372 func (lc *LocalConfig) ReadFrom(r io.Reader) (int64, error) { 373 return 1, xml.NewDecoder(r).Decode(lc) 374 } 375 376 func LocalConfigFromFile(jirix *jiri.X, filename string) (LocalConfig, error) { 377 var lc LocalConfig 378 f, err := os.Open(filename) 379 if os.IsNotExist(err) { 380 return lc, nil 381 } else if err != nil { 382 return lc, fmtError(err) 383 } 384 _, err = lc.ReadFrom(f) 385 return lc, err 386 } 387 388 // Writes the localConfig to given writer. Returns incorrect bytes 389 func (lc *LocalConfig) WriteTo(writer io.Writer) (int64, error) { 390 encoder := xml.NewEncoder(writer) 391 encoder.Indent("", " ") 392 return 1, encoder.Encode(lc) 393 } 394 395 func (lc *LocalConfig) ToFile(jirix *jiri.X, filename string) error { 396 if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil { 397 return fmtError(err) 398 } 399 writer, err := os.Create(filename) 400 if err != nil { 401 return fmtError(err) 402 } 403 defer writer.Close() 404 _, err = lc.WriteTo(writer) 405 return err 406 } 407 408 func WriteLocalConfig(jirix *jiri.X, project Project, lc LocalConfig) error { 409 configFile := filepath.Join(project.Path, jiri.ProjectMetaDir, jiri.ProjectConfigFile) 410 return lc.ToFile(jirix, configFile) 411 } 412 413 // Hook represents a hook to run 414 type Hook struct { 415 Name string `xml:"name,attr"` 416 Action string `xml:"action,attr"` 417 ProjectName string `xml:"project,attr"` 418 XMLName struct{} `xml:"hook"` 419 ActionPath string `xml:"-"` 420 } 421 422 // HookKey is a unique string for a project. 423 type HookKey string 424 425 type Hooks map[HookKey]Hook 426 427 // Key returns the unique HookKey for the hook. 428 func (h Hook) Key() HookKey { 429 return MakeHookKey(h.Name, h.ProjectName) 430 } 431 432 // MakeHookKey returns the hook key, given the hook and project name. 433 func MakeHookKey(name, projectName string) HookKey { 434 return HookKey(name + KeySeparator + projectName) 435 } 436 437 func (h *Hook) validate() error { 438 if strings.Contains(h.Name, KeySeparator) { 439 return fmt.Errorf("bad hook: name cannot contain %q: %+v", KeySeparator, *h) 440 } 441 if strings.Contains(h.ProjectName, KeySeparator) { 442 return fmt.Errorf("bad hook: project cannot contain %q: %+v", KeySeparator, *h) 443 } 444 return nil 445 } 446 447 // HooksByName implements the Sort interface. It sorts Hooks by the Name 448 // and ProjectName field. 449 type HooksByName []Hook 450 451 func (hooks HooksByName) Len() int { 452 return len(hooks) 453 } 454 func (hooks HooksByName) Swap(i, j int) { 455 hooks[i], hooks[j] = hooks[j], hooks[i] 456 } 457 func (hooks HooksByName) Less(i, j int) bool { 458 if hooks[i].Name == hooks[j].Name { 459 return hooks[i].ProjectName < hooks[j].ProjectName 460 } 461 return hooks[i].Name < hooks[j].Name 462 } 463 464 // Package struct represents the <package> tag in manifest files. 465 type Package struct { 466 // Name represents the remote cipd path of the package. 467 Name string `xml:"name,attr"` 468 469 // Version represents the version tag of the cipd package. 470 Version string `xml:"version,attr"` 471 472 // Path stores the local path of fetched cipd package. 473 Path string `xml:"path,attr,omitempty"` 474 475 // Internal marks if this package require special permission 476 // for access 477 Internal bool `xml:"internal,attr,omitempty"` 478 479 // Platforms defines the available platforms for this cipd package. 480 Platforms string `xml:"platforms,attr,omitempty"` 481 482 // Flag defines the content that should be written to a file when 483 // this package is successfully fetched. 484 Flag string `xml:"flag,attr,omitempty"` 485 486 // Attributes store the the list attributes for this package. 487 // When it starts with "+", a computed default attributes will 488 // be appended. 489 Attributes string `xml:"attributes,attr,omitempty"` 490 491 // Instances store the known instance ids for this package. 492 // It is mainly used by snapshot file. 493 Instances []PackageInstance `xml:"instance"` 494 XMLName struct{} `xml:"package"` 495 496 // ComputedAttributes stores computed attributes object 497 // which is easiler to perform matching and comparing. 498 ComputedAttributes attributes `xml:"-"` 499 500 // ManifestPath stores the absolute path of the manifest. 501 ManifestPath string `xml:"-"` 502 } 503 504 type PackageKey string 505 506 type Packages map[PackageKey]Package 507 508 type PackageKeys []PackageKey 509 510 func (p Package) Key() PackageKey { 511 return PackageKey(p.Path + KeySeparator + p.Name) 512 } 513 514 func (pks PackageKeys) Len() int { return len(pks) } 515 func (pks PackageKeys) Less(i, j int) bool { return string(pks[i]) < string(pks[j]) } 516 func (pks PackageKeys) Swap(i, j int) { pks[i], pks[j] = pks[j], pks[i] } 517 518 // FilterACL returns a new Packages map without any inaccessible packages. 519 func (p *Packages) FilterACL(jirix *jiri.X) (Packages, bool, error) { 520 // Perform ACL checks on internal projects 521 pkgACLMap := make(map[string]bool) 522 hasInternal := false 523 for _, pkg := range *p { 524 pkg.Name = strings.TrimRight(pkg.Name, "/") 525 if pkg.Internal { 526 hasInternal = true 527 pkgACLMap[pkg.Name] = false 528 } 529 } 530 if len(pkgACLMap) != 0 { 531 if err := cipd.CheckPackageACL(jirix, pkgACLMap); err != nil { 532 return nil, false, err 533 } 534 } 535 retPkgs := make(Packages) 536 for _, pkg := range *p { 537 if val, ok := pkgACLMap[pkg.Name]; ok && !val { 538 continue 539 } 540 retPkgs[pkg.Key()] = pkg 541 } 542 return retPkgs, hasInternal, nil 543 } 544 545 type PackageInstance struct { 546 Name string `xml:"name,attr"` 547 ID string `xml:"id,attr"` 548 XMLName struct{} `xml:"instance"` 549 } 550 551 // FillDefaults function fills default platforms information into 552 // Package struct if it is not defined and path is using template. 553 func (p *Package) FillDefaults() error { 554 if cipd.MustExpand(p.Name) && p.Platforms == "" { 555 for _, v := range cipd.DefaultPlatforms() { 556 p.Platforms += v.String() + "," 557 } 558 if p.Platforms[len(p.Platforms)-1] == ',' { 559 p.Platforms = p.Platforms[:len(p.Platforms)-1] 560 } 561 } 562 return nil 563 } 564 565 // GetPath returns the relative path that Package p should be 566 // downloaded to. 567 func (p *Package) GetPath() (string, error) { 568 if p.Path == "" { 569 cipdPath := p.Name 570 // Replace template with current platform information. 571 // If failed, skip filling in default path. 572 if cipd.MustExpand(cipdPath) { 573 expanded, err := cipd.Expand(cipdPath, []cipd.Platform{cipd.CipdPlatform}) 574 if err != nil { 575 return "", err 576 } 577 if len(expanded) > 0 { 578 cipdPath = expanded[0] 579 } 580 } 581 if !cipd.MustExpand(cipdPath) { 582 base := path.Base(cipdPath) 583 if _, err := cipd.NewPlatform(base); err == nil { 584 // base is the name for a platform 585 base = filepath.Join(path.Base(path.Dir(cipdPath)), base) 586 } 587 return filepath.Join("prebuilt", base), nil 588 } 589 return "prebuilt", nil 590 } 591 return p.Path, nil 592 } 593 594 // GetPlatforms returns the platforms information of 595 // this Package struct. 596 func (p *Package) GetPlatforms() ([]cipd.Platform, error) { 597 if err := p.FillDefaults(); err != nil { 598 return nil, err 599 } 600 retList := make([]cipd.Platform, 0) 601 platStrs := strings.Split(p.Platforms, ",") 602 for _, platStr := range platStrs { 603 if platStr == "" { 604 continue 605 } 606 plat, err := cipd.NewPlatform(platStr) 607 if err != nil { 608 return nil, err 609 } 610 retList = append(retList, plat) 611 } 612 return retList, nil 613 } 614 615 // LoadManifest loads the manifest, starting with the .jiri_manifest file, 616 // resolving remote and local imports. Returns the projects specified by 617 // the manifest. 618 // 619 // WARNING: LoadManifest cannot be run multiple times in parallel! It invokes 620 // git operations which require a lock on the filesystem. If you see errors 621 // about ".git/index.lock exists", you are likely calling LoadManifest in 622 // parallel. 623 func LoadManifest(jirix *jiri.X) (Projects, Hooks, Packages, error) { 624 jirix.TimerPush("load manifest") 625 defer jirix.TimerPop() 626 file := jirix.JiriManifestFile() 627 localProjects, err := LocalProjects(jirix, FastScan) 628 if err != nil { 629 return nil, nil, nil, err 630 } 631 return LoadManifestFile(jirix, file, localProjects, false) 632 } 633 634 func (ld *loader) warnOverrides(jirix *jiri.X) { 635 if len(ld.ProjectOverrides) != 0 { 636 for _, v := range ld.ProjectOverrides { 637 revision := v.Revision 638 if revision == "" { 639 revision = "HEAD" 640 } 641 jirix.Logger.Warningf("Project %s(remote: %s) is pinned to revision %s, if that is not what you want, please run \"jiri override --delete %s %s\" to unpin it.", v.Name, v.Remote, revision, v.Name, v.Remote) 642 jirix.OverrideWarned = true 643 } 644 } 645 if len(ld.ImportOverrides) != 0 { 646 for _, v := range ld.ImportOverrides { 647 revision := v.Revision 648 if revision == "" { 649 revision = "HEAD" 650 } 651 jirix.Logger.Warningf("Import %s(remote: %s) is pinned to revision %s, if that is not what you want, please run \"jiri override --import-manifest=%s --delete %s %s\" to unpin it.", v.Name, v.Remote, revision, v.Manifest, v.Name, v.Remote) 652 jirix.OverrideWarned = true 653 } 654 } 655 } 656 657 func (ld *loader) enforceLocks(jirix *jiri.X) error { 658 enforceProjLocks := func(jirix *jiri.X) (err error) { 659 for _, v := range ld.Projects { 660 if projectLock, ok := ld.ProjectLocks[ProjectLockKey(v.Key())]; ok { 661 if v.Revision == "" || v.Revision == "HEAD" { 662 v.Revision = projectLock.Revision 663 ld.Projects[v.Key()] = v 664 } else if v.Revision != projectLock.Revision { 665 s := fmt.Sprintf("project %+v has conflicting revisions in manifest and jiri.lock: %s:%s", v, v.Revision, projectLock.Revision) 666 jirix.Logger.Debugf(s) 667 err = errors.New(s) 668 } 669 } 670 } 671 return 672 } 673 674 enforcePkgLocks := func(jirix *jiri.X) (err error) { 675 usedPkgLocks := make(map[PackageLockKey]bool) 676 for k := range ld.PackageLocks { 677 usedPkgLocks[k] = false 678 } 679 for _, v := range ld.Packages { 680 plats, err := v.GetPlatforms() 681 if err != nil { 682 return err 683 } 684 pkgs, err := cipd.Expand(v.Name, plats) 685 if err != nil { 686 return err 687 } 688 for _, pkg := range pkgs { 689 if pkgLock, ok := ld.PackageLocks[PackageLockKey(pkg+KeySeparator+v.Version)]; ok { 690 if pkgLock.VersionTag != v.Version && !jirix.IgnoreLockConflicts { 691 // Package version conflicts detected. Treated it as an error. 692 s := fmt.Sprintf("package %q has conflicting version in manifest and jiri.lock: %s:%s", v.Name, v.Version, pkgLock.VersionTag) 693 jirix.Logger.Debugf(s) 694 err = errors.New(s) 695 } 696 ins := PackageInstance{ 697 Name: pkgLock.PackageName, 698 ID: pkgLock.InstanceID, 699 } 700 v.Instances = append(v.Instances, ins) 701 ld.Packages[v.Key()] = v 702 usedPkgLocks[pkgLock.Key()] = true 703 } else { 704 jirix.Logger.Debugf("Package %q is not found in jiri.lock", pkg) 705 } 706 } 707 if err != nil { 708 return err 709 } 710 } 711 for k, v := range usedPkgLocks { 712 if !v { 713 jirix.Logger.Debugf("PackageLock %v is not used", k) 714 } 715 } 716 return 717 } 718 719 if err := enforceProjLocks(jirix); err != nil { 720 return err 721 } 722 723 if err := enforcePkgLocks(jirix); err != nil { 724 return err 725 } 726 return nil 727 } 728 729 // LoadManifestFile loads the manifest starting with the given file, resolving 730 // remote and local imports. Local projects are used to resolve remote imports; 731 // if nil, encountering any remote import will result in an error. 732 // 733 // WARNING: LoadManifestFile cannot be run multiple times in parallel! It 734 // invokes git operations which require a lock on the filesystem. If you see 735 // errors about ".git/index.lock exists", you are likely calling 736 // LoadManifestFile in parallel. 737 func LoadManifestFile(jirix *jiri.X, file string, localProjects Projects, localManifest bool) (Projects, Hooks, Packages, error) { 738 ld := newManifestLoader(localProjects, false, file) 739 if err := ld.Load(jirix, "", "", file, "", "", "", localManifest); err != nil { 740 return nil, nil, nil, err 741 } 742 jirix.AddCleanupFunc(ld.cleanup) 743 if jirix.LockfileEnabled { 744 if err := ld.enforceLocks(jirix); err != nil { 745 return nil, nil, nil, err 746 } 747 } 748 if !jirix.OverrideWarned { 749 ld.warnOverrides(jirix) 750 } 751 ld.GenerateGitAttributesForProjects(jirix) 752 return ld.Projects, ld.Hooks, ld.Packages, nil 753 } 754 755 // LoadUpdatedManifest loads an updated manifest starting with the .jiri_manifest file for localProjects. It will use 756 // local manifest files instead of manifest files in remote repositories if localManifest is set to true. 757 func LoadUpdatedManifest(jirix *jiri.X, localProjects Projects, localManifest bool) (Projects, Hooks, Packages, error) { 758 jirix.TimerPush("load updated manifest") 759 defer jirix.TimerPop() 760 ld := newManifestLoader(localProjects, true, jirix.JiriManifestFile()) 761 if err := ld.Load(jirix, "", "", jirix.JiriManifestFile(), "", "", "", localManifest); err != nil { 762 return nil, nil, nil, err 763 } 764 jirix.AddCleanupFunc(ld.cleanup) 765 if jirix.LockfileEnabled { 766 if err := ld.enforceLocks(jirix); err != nil { 767 return nil, nil, nil, err 768 } 769 } 770 if !jirix.OverrideWarned { 771 ld.warnOverrides(jirix) 772 } 773 ld.GenerateGitAttributesForProjects(jirix) 774 return ld.Projects, ld.Hooks, ld.Packages, nil 775 } 776 777 // ResolveImplicitPackageVersions resolves the version field of packages if it 778 // pins to a project's revision hash 779 func ResolveImplicitPackageVersions(jirix *jiri.X, projects Projects, pkgs Packages) (Packages, error) { 780 // Example: 781 // <package name="fuchsia/dart-sdk/${platform}" 782 // path="third_party/dart/tools/sdks/dart-sdk" 783 // version="git_revision:{{(index .Projects "dart/sdk").Revision}}"/> 784 // The fuchsia/dart-sdk package is pinned to current dart/sdk project's Revision 785 templateRE := regexp.MustCompile(`{{[^}]*}}`) 786 var projMap struct { 787 Projects map[string]Project 788 } 789 retPkgs := make(Packages) 790 for k, v := range pkgs { 791 retPkgs[k] = v 792 } 793 projMap.Projects = make(map[string]Project) 794 for _, proj := range projects { 795 if v, ok := projMap.Projects[proj.Name]; ok { 796 // Just a warning since jiri could handle projects with duplicated names. 797 jirix.Logger.Warningf("Found more than 1 projects have the same name: %+v:%+v", proj, v) 798 } 799 projMap.Projects[proj.Name] = proj 800 } 801 802 for _, pkg := range pkgs { 803 if !templateRE.MatchString(pkg.Version) { 804 continue 805 } 806 tpl, err := template.New("version").Parse(pkg.Version) 807 if err != nil { 808 return nil, err 809 } 810 var verBuf bytes.Buffer 811 if err := tpl.Execute(&verBuf, &projMap); err != nil { 812 return nil, err 813 } 814 pkg.Version = verBuf.String() 815 retPkgs[pkg.Key()] = pkg 816 } 817 return retPkgs, nil 818 } 819 820 // resovlePackageLocks resolves instance ids using versions described in given 821 // pkgs using cipd. 822 func resolvePackageLocks(jirix *jiri.X, projects Projects, pkgs Packages) (PackageLocks, error) { 823 jirix.TimerPush("resolve instance id for cipd packages") 824 defer jirix.TimerPop() 825 826 pkgs, _, err := pkgs.FilterACL(jirix) 827 if err != nil { 828 return nil, err 829 } 830 831 ensureFilePath, err := generateEnsureFile(jirix, projects, pkgs, false) 832 if err != nil { 833 return nil, err 834 } 835 defer os.Remove(ensureFilePath) 836 837 pkgInstances, err := cipd.Resolve(jirix, ensureFilePath) 838 if err != nil { 839 return nil, err 840 } 841 // TODO: Remove this boilerplate once we have a better package 842 // layout that doesn't cause import cycles 843 pkgLocks := make(PackageLocks) 844 for _, val := range pkgInstances { 845 pkgLock := PackageLock{ 846 PackageName: val.PackageName, 847 VersionTag: val.VersionTag, 848 InstanceID: val.InstanceID, 849 } 850 pkgLocks[pkgLock.Key()] = pkgLock 851 } 852 853 return pkgLocks, nil 854 } 855 856 // resolveProjectLocks resolves project revisions <project> tags in manifests 857 func resolveProjectLocks(jirix *jiri.X, projects Projects) (ProjectLocks, error) { 858 projectLocks := make(ProjectLocks) 859 for _, v := range projects { 860 projectLock := ProjectLock{v.Remote, v.Name, v.Revision} 861 projectLocks[projectLock.Key()] = projectLock 862 } 863 return projectLocks, nil 864 } 865 866 // FetchPackages fetches prebuilt packages described in given pkgs using cipd. 867 // Parameter fetchTimeout is in minutes. 868 func FetchPackages(jirix *jiri.X, projects Projects, pkgs Packages, fetchTimeout uint) error { 869 jirix.TimerPush("fetch cipd packages") 870 defer jirix.TimerPop() 871 872 pkgsWAccess, hasInternalPkgs, err := pkgs.FilterACL(jirix) 873 if err != nil { 874 return err 875 } 876 877 ensureFilePath, err := generateEnsureFile(jirix, projects, pkgsWAccess, !jirix.LockfileEnabled || jirix.UsingSnapshot) 878 if err != nil { 879 return err 880 } 881 defer os.Remove(ensureFilePath) 882 883 if jirix.LockfileEnabled && !jirix.UsingSnapshot { 884 versionFilePath, err := generateVersionFile(jirix, ensureFilePath, pkgs) 885 if err != nil { 886 return err 887 } 888 defer os.Remove(versionFilePath) 889 } 890 891 if err := cipd.Ensure(jirix, ensureFilePath, jirix.Root, fetchTimeout); err != nil { 892 return err 893 } 894 895 if hasInternalPkgs { 896 if err := writePackageJSON(jirix, len(pkgs) == len(pkgsWAccess)); err != nil { 897 return err 898 } 899 } 900 901 // Write explict flags. 902 if err := WritePackageFlags(jirix, pkgs, pkgsWAccess); err != nil { 903 return err 904 } 905 906 if len(pkgs) > len(pkgsWAccess) { 907 cipdLoggedIn, err := cipd.CheckLoggedIn(jirix) 908 if err != nil { 909 return err 910 } 911 if !cipdLoggedIn { 912 jirix.Logger.Warningf("Some packages are skipped by cipd due to lack of access, you might want to run \"%s auth-login\" and try again", jirix.CIPDPath()) 913 } 914 } 915 return nil 916 } 917 918 // WritePackageFlags write flag files into project directory using in "flag" 919 // attribute from pkgs. 920 func WritePackageFlags(jirix *jiri.X, pkgs, pkgsWA Packages) error { 921 // The flag attribute has a format of $FILE_NAME|$FLAG_SUCCESSFUL|$FLAG_FAILED 922 // When a package is successfully downloaded, jiri will write $FLAG_SUCCESSFUL 923 // to $FILE_NAME. If the package is not downloaded due to access reasons, 924 // jiri will write $FLAG_FAILED to $FILE_NAME. 925 // '|' is a forbidden symbol in Windows path, which is unlikely 926 // to be used by path. 927 928 flagMap := make(map[string]string) 929 fill := func(file, flag string) error { 930 if v, ok := flagMap[file]; ok { 931 if v != flag { 932 return fmt.Errorf("encountered conflicting flags for file %q: %q conflicts with %q", file, v, flag) 933 } 934 } else { 935 flagMap[file] = flag 936 } 937 return nil 938 } 939 940 for k, v := range pkgs { 941 if v.Flag == "" { 942 continue 943 } 944 fields := strings.Split(v.Flag, "|") 945 if len(fields) != 3 { 946 return fmt.Errorf("unknown package flag format found in package %+v", v) 947 } 948 if _, ok := pkgsWA[k]; ok { 949 // package is successfully fetched, write successful flag. 950 if err := fill(fields[0], fields[1]); err != nil { 951 return err 952 } 953 } else { 954 // package is failed to fetched, write failed flag. 955 if err := fill(fields[0], fields[2]); err != nil { 956 return err 957 } 958 } 959 } 960 961 var writeErrorBuf bytes.Buffer 962 for k, v := range flagMap { 963 if err := ioutil.WriteFile(filepath.Join(jirix.Root, k), []byte(v), 0644); err != nil { 964 writeErrorBuf.WriteString(fmt.Sprintf("write package flag %q to file %q failed due to error: %v\n", v, k, err)) 965 } 966 } 967 if writeErrorBuf.Len() > 0 { 968 return errors.New(writeErrorBuf.String()) 969 } 970 return nil 971 } 972 973 // GenerateJSON generates a json file which contains fetched 974 // packages. 975 func writePackageJSON(jirix *jiri.X, access bool) error { 976 var internalAccess struct { 977 Access bool `json:"internal_access"` 978 } 979 internalAccess.Access = access 980 jsonData, err := json.MarshalIndent(&internalAccess, "", " ") 981 if err != nil { 982 return err 983 } 984 if jirix.PrebuiltJSON == "" { 985 // Skip json file creation if PrebuiltJSON is not set. 986 return nil 987 } 988 return ioutil.WriteFile(filepath.Join(jirix.RootMetaDir(), jirix.PrebuiltJSON), jsonData, 0644) 989 } 990 991 func generateEnsureFile(jirix *jiri.X, projects Projects, pkgs Packages, ignoreCryptoCheck bool) (string, error) { 992 pkgs, err := ResolveImplicitPackageVersions(jirix, projects, pkgs) 993 if err != nil { 994 return "", err 995 } 996 ensureFile, err := ioutil.TempFile("", "jiri*.ensure") 997 if err != nil { 998 return "", fmt.Errorf("not able to create tmp file: %v", err) 999 } 1000 defer ensureFile.Close() 1001 ensureFilePath := ensureFile.Name() 1002 1003 // Write header information 1004 // TODO: add "verfy_platform" attribute to each package tag 1005 // to avoid hardcoding platform names in Jiri 1006 var ensureFileBuf bytes.Buffer 1007 if !ignoreCryptoCheck { 1008 // Collect platforms used by this project 1009 allPlats := make(map[string]cipd.Platform) 1010 // CIPD ensure-file-resolve requires $VerifiedPlatform to be present 1011 // even if the package name is not using ${platform} template. 1012 // Put DefaultPlatforms into header to walkaround this issue. 1013 for _, plat := range cipd.DefaultPlatforms() { 1014 allPlats[plat.String()] = plat 1015 } 1016 for _, pkg := range pkgs { 1017 plats, err := pkg.GetPlatforms() 1018 if err != nil { 1019 return "", err 1020 } 1021 for _, plat := range plats { 1022 allPlats[plat.String()] = plat 1023 } 1024 } 1025 1026 for _, plat := range allPlats { 1027 ensureFileBuf.WriteString(fmt.Sprintf("$VerifiedPlatform %s\n", plat)) 1028 } 1029 versionFileName := ensureFilePath[:len(ensureFilePath)-len(".ensure")] + ".version" 1030 ensureFileBuf.WriteString("$ResolvedVersions " + versionFileName + "\n") 1031 } 1032 if jirix.CipdParanoidMode { 1033 ensureFileBuf.WriteString("$ParanoidMode CheckPresence\n") 1034 } 1035 ensureFileBuf.WriteString("\n") 1036 1037 for _, pkg := range pkgs { 1038 1039 cipdDecl, err := pkg.cipdDecl(jirix) 1040 if err != nil { 1041 return "", err 1042 } 1043 ensureFileBuf.WriteString(cipdDecl) 1044 ensureFileBuf.WriteString("\n") 1045 } 1046 1047 jirix.Logger.Debugf("Generated ensure file content:\n%v", ensureFileBuf.String()) 1048 if _, err := ensureFileBuf.WriteTo(ensureFile); err != nil { 1049 return "", err 1050 } 1051 if err := ensureFile.Sync(); err != nil { 1052 return "", err 1053 } 1054 1055 return ensureFilePath, nil 1056 } 1057 1058 func (p *Package) cipdDecl(jirix *jiri.X) (string, error) { 1059 var buf bytes.Buffer 1060 // Write "@Subdir" line to cipd declaration 1061 subdir, err := p.GetPath() 1062 if err != nil { 1063 return "", err 1064 } 1065 tmpl, err := template.New("pack").Parse(subdir) 1066 if err != nil { 1067 return "", fmt.Errorf("parsing package path %q failed", subdir) 1068 } 1069 var subdirBuf bytes.Buffer 1070 // subdir is using fuchsia platform format instead of 1071 // using cipd platform format 1072 tmpl.Execute(&subdirBuf, cipd.FuchsiaPlatform(cipd.CipdPlatform)) 1073 subdir = subdirBuf.String() 1074 buf.WriteString(fmt.Sprintf("@Subdir %s\n", subdir)) 1075 // Write package version line to cipd declaration 1076 plats, err := p.GetPlatforms() 1077 if err != nil { 1078 return "", err 1079 } 1080 var cipdPath, version string 1081 version = p.Version 1082 cipdPath, err = cipd.Decl(p.Name, plats) 1083 if err != nil { 1084 return "", err 1085 } 1086 if jirix.UsingSnapshot && len(p.Instances) != 0 { 1087 candPath, err := cipd.Expand(p.Name, []cipd.Platform{cipd.CipdPlatform}) 1088 if err != nil { 1089 return "", err 1090 } 1091 if len(candPath) > 0 { 1092 for _, inst := range p.Instances { 1093 if inst.Name == candPath[0] { 1094 cipdPath = candPath[0] 1095 version = inst.ID 1096 break 1097 } 1098 } 1099 } else { 1100 // cipd.Expand failed expand 1101 // This may happen if the cipdPath does not allow platform 1102 // in cipd.CipdPlatform. E.g. 1103 // "example/linux-${arch=amd64}" expanded with "linux-arm64". 1104 // Leave a log in Debug log. 1105 jirix.Logger.Debugf("cipd.Expand failed to expand cipd path %q using platforms %v", p.Name, cipd.CipdPlatform) 1106 } 1107 } 1108 buf.WriteString(fmt.Sprintf("%s %s\n", cipdPath, version)) 1109 return buf.String(), nil 1110 } 1111 1112 func generateVersionFile(jirix *jiri.X, ensureFile string, pkgs Packages) (string, error) { 1113 versionFileName := ensureFile[:len(ensureFile)-len(".ensure")] + ".version" 1114 1115 var versionFileBuf bytes.Buffer 1116 // Just pour everything in pkgLocks into version file without matching package 1117 // names. cipd will do the matching for us. 1118 for _, pkg := range pkgs { 1119 jirix.Logger.Debugf("Generate version file using %+v", pkg) 1120 for _, ins := range pkg.Instances { 1121 decl := fmt.Sprintf("\n%s\n\t%s\n\t%s\n", ins.Name, pkg.Version, ins.ID) 1122 versionFileBuf.WriteString(decl) 1123 } 1124 } 1125 jirix.Logger.Debugf("Generated version file content:\n%v", versionFileBuf.String()) 1126 return versionFileName, ioutil.WriteFile(versionFileName, versionFileBuf.Bytes(), 0655) 1127 } 1128 1129 // RunHooks runs all given hooks. 1130 func RunHooks(jirix *jiri.X, hooks Hooks, runHookTimeout uint) error { 1131 jirix.TimerPush("run hooks") 1132 defer jirix.TimerPop() 1133 jirix.Logger.Debugf("Running Jiri hooks") 1134 defer jirix.Logger.Debugf("Running Jiri ") 1135 type result struct { 1136 outFile *os.File 1137 errFile *os.File 1138 err error 1139 } 1140 ch := make(chan result) 1141 tmpDir, err := ioutil.TempDir("", "run-hooks") 1142 if err != nil { 1143 return fmt.Errorf("not able to create tmp dir: %v", err) 1144 } 1145 defer os.RemoveAll(tmpDir) 1146 for _, hook := range hooks { 1147 go func(hook Hook) { 1148 logStr := fmt.Sprintf("running hook(%s) for project %q", hook.Name, hook.ProjectName) 1149 jirix.Logger.Debugf(logStr) 1150 task := jirix.Logger.AddTaskMsg(logStr) 1151 defer task.Done() 1152 outFile, err := ioutil.TempFile(tmpDir, hook.Name+"-out") 1153 if err != nil { 1154 ch <- result{nil, nil, fmtError(err)} 1155 return 1156 } 1157 errFile, err := ioutil.TempFile(tmpDir, hook.Name+"-err") 1158 if err != nil { 1159 ch <- result{nil, nil, fmtError(err)} 1160 return 1161 } 1162 1163 fmt.Fprintf(outFile, "output for hook(%v) for project %q\n", hook.Name, hook.ProjectName) 1164 fmt.Fprintf(errFile, "Error for hook(%v) for project %q\n", hook.Name, hook.ProjectName) 1165 cmdLine := filepath.Join(hook.ActionPath, hook.Action) 1166 err = retry.Function(jirix, func() error { 1167 ctx, cancel := context.WithTimeout(context.Background(), time.Duration(runHookTimeout)*time.Minute) 1168 defer cancel() 1169 command := exec.CommandContext(ctx, cmdLine) 1170 command.Dir = hook.ActionPath 1171 command.Stdin = os.Stdin 1172 command.Stdout = outFile 1173 command.Stderr = errFile 1174 env := jirix.Env() 1175 command.Env = envvar.MapToSlice(env) 1176 jirix.Logger.Tracef("Run: %q", cmdLine) 1177 err = command.Run() 1178 if ctx.Err() == context.DeadlineExceeded { 1179 err = ctx.Err() 1180 } 1181 scm := gitutil.New(jirix, gitutil.RootDirOpt(filepath.Dir(filepath.Dir(cmdLine)))) 1182 revision, err2 := scm.CurrentRevisionOfBranch("HEAD") 1183 if err2 == nil { 1184 jirix.Logger.Debugf(" Invoked hook(%v) for project %q on revision %q", hook.Name, hook.ProjectName, revision) 1185 } 1186 return err 1187 }, fmt.Sprintf("running hook(%s) for project %s", hook.Name, hook.ProjectName), 1188 retry.AttemptsOpt(jirix.Attempts)) 1189 ch <- result{outFile, errFile, err} 1190 }(hook) 1191 1192 } 1193 1194 err = nil 1195 timeout := false 1196 for range hooks { 1197 out := <-ch 1198 defer func() { 1199 if out.outFile != nil { 1200 out.outFile.Close() 1201 } 1202 if out.errFile != nil { 1203 out.errFile.Close() 1204 } 1205 }() 1206 if out.err == context.DeadlineExceeded { 1207 timeout = true 1208 out.outFile.Sync() 1209 out.outFile.Seek(0, 0) 1210 var buf bytes.Buffer 1211 io.Copy(&buf, out.outFile) 1212 jirix.Logger.Errorf("Timeout while executing hook\n%s\n\n", buf.String()) 1213 err = fmt.Errorf("Hooks execution failed.") 1214 continue 1215 } 1216 var outBuf bytes.Buffer 1217 if out.outFile != nil { 1218 out.outFile.Sync() 1219 out.outFile.Seek(0, 0) 1220 io.Copy(&outBuf, out.outFile) 1221 } 1222 if out.err != nil { 1223 var buf bytes.Buffer 1224 if out.errFile != nil { 1225 out.errFile.Sync() 1226 out.errFile.Seek(0, 0) 1227 io.Copy(&buf, out.errFile) 1228 } 1229 jirix.Logger.Errorf("%s\n%s\n%s\n", out.err, buf.String(), outBuf.String()) 1230 err = fmt.Errorf("Hooks execution failed.") 1231 } else { 1232 if outBuf.String() != "" { 1233 jirix.Logger.Debugf("%s\n", outBuf.String()) 1234 } 1235 } 1236 } 1237 if timeout { 1238 err = fmt.Errorf("%s Use %s flag to set timeout.", err, jirix.Color.Yellow("-hook-timeout")) 1239 } 1240 return err 1241 } 1242 1243 type commitMsgFetcher map[string][]byte 1244 1245 func (f commitMsgFetcher) fetch(jirix *jiri.X, gerritHost, path string) ([]byte, error) { 1246 bytes, ok := f[gerritHost] 1247 if !ok { 1248 jirix.Logger.Debugf("Fetching %q", gerritHost+"/tools/hooks/commit-msg") 1249 data, err := gerrit.FetchFile(gerritHost, "/tools/hooks/commit-msg") 1250 if err != nil { 1251 if err != gerrit.ErrRedirectOnGerrit { 1252 // Network or disk IO error, halt jiri 1253 return nil, err 1254 } 1255 // gerritHost require SSO login 1256 if jirix.RewriteSsoToHttps { 1257 // Gerrit host require SSO but jiri has rewritesso flag turned on 1258 // In this case git hooks are useless, stop fetching git hooks 1259 return nil, errGitHookNotRequired 1260 } 1261 1262 // Use commit-msg in cache if the domain has same eTLD and SLD. 1263 for k, v := range f { 1264 urlK, err := url.Parse(k) 1265 if err != nil { 1266 // This should not happen as this url is already downloaded before. 1267 return nil, fmt.Errorf("download commit-msg hook for host %q failed due to error %v", gerritHost, err) 1268 } 1269 urlG, err := url.Parse(gerritHost) 1270 if err != nil { 1271 // This should not happen either as gerritHost will be parsed by gerrit.FetchFile 1272 return nil, fmt.Errorf("download commit-msg hook from host %q failed due to error %v", gerritHost, err) 1273 } 1274 etpoK, err := publicsuffix.EffectiveTLDPlusOne(urlK.Hostname()) 1275 if err != nil { 1276 // This should not happen as Both SLD and TLD should exist. 1277 return nil, fmt.Errorf("download commit-msg hook from host %q failed due to error %v", gerritHost, err) 1278 } 1279 etpoG, err := publicsuffix.EffectiveTLDPlusOne(urlG.Hostname()) 1280 if err != nil { 1281 // This should not happen as Both SLD and TLD should exist. 1282 return nil, fmt.Errorf("download commit-msg hook from host %q failed due to error %v", gerritHost, err) 1283 } 1284 1285 if etpoK == etpoG { 1286 jirix.Logger.Debugf("use commit-msg hook from host %q for host %q due to access limitations", k, gerritHost) 1287 data = v 1288 err = nil 1289 break 1290 } 1291 } 1292 1293 if data == nil { 1294 // Could not find commit-msg in cache from domains with same eTLD and SLD. 1295 // Fetch commit-msg from fuchsia's gerrit server. 1296 data, err = gerrit.FetchFile(fuchsiaGerritHost, "/tools/hooks/commit-msg") 1297 if err != nil { 1298 // This will only happen if configuration error occured on fuchsia gerrit server 1299 return nil, fmt.Errorf("download commit-msg hook from host %q failed due to error %v", fuchsiaGerritHost, err) 1300 } 1301 jirix.Logger.Debugf("fallback to commit-msg from host %q for host %q due to access limitations", fuchsiaGerritHost, gerritHost) 1302 } 1303 } 1304 f[gerritHost] = data 1305 return data, nil 1306 } 1307 jirix.Logger.Debugf("Cached %q", gerritHost+"/tools/hooks/commit-msg") 1308 return bytes, nil 1309 } 1310 1311 func applyGitHooks(jirix *jiri.X, ops []operation) error { 1312 jirix.TimerPush("apply githooks") 1313 defer jirix.TimerPop() 1314 commitMsgFetcher := commitMsgFetcher{} 1315 for _, op := range ops { 1316 if op.Kind() != "delete" && !op.Project().LocalConfig.Ignore && !op.Project().LocalConfig.NoUpdate { 1317 if op.Project().GerritHost != "" { 1318 hookPath := filepath.Join(op.Project().Path, ".git", "hooks", "commit-msg") 1319 commitHook, err := os.Create(hookPath) 1320 if err != nil { 1321 return fmtError(err) 1322 } 1323 bytes, err := commitMsgFetcher.fetch(jirix, op.Project().GerritHost, "/tools/hooks/commit-msg") 1324 if err != nil { 1325 if err != errGitHookNotRequired { 1326 jirix.Logger.Debugf("%v", err) 1327 } 1328 commitHook.Close() 1329 os.Remove(hookPath) 1330 continue 1331 } 1332 1333 if _, err := commitHook.Write(bytes); err != nil { 1334 return err 1335 } 1336 jirix.Logger.Debugf("Saved commit-msg hook to project %q", op.Project().Path) 1337 commitHook.Close() 1338 if err := os.Chmod(hookPath, 0750); err != nil { 1339 return fmtError(err) 1340 } 1341 } 1342 hookPath := filepath.Join(op.Project().Path, ".git", "hooks", "post-commit") 1343 commitHook, err := os.Create(hookPath) 1344 if err != nil { 1345 return err 1346 } 1347 bytes := []byte(`#!/bin/sh 1348 1349 if ! git symbolic-ref HEAD &> /dev/null; then 1350 echo -e "WARNING: You are in a detached head state! You might lose this commit.\nUse 'git checkout -b <branch> to put it on a branch.\n" 1351 fi 1352 `) 1353 if _, err := commitHook.Write(bytes); err != nil { 1354 return err 1355 } 1356 commitHook.Close() 1357 if err := os.Chmod(hookPath, 0750); err != nil { 1358 return err 1359 } 1360 } 1361 if op.Project().GitHooks == "" { 1362 continue 1363 } 1364 // Don't want to run hooks when repo is deleted 1365 if op.Kind() == "delete" { 1366 continue 1367 } 1368 // Apply git hooks, overwriting any existing hooks. Jiri is in control of 1369 // writing all hooks. 1370 gitHooksDstDir := filepath.Join(op.Project().Path, ".git", "hooks") 1371 // Copy the specified GitHooks directory into the project's git 1372 // hook directory. We walk the file system, creating directories 1373 // and copying files as we encounter them. 1374 copyFn := func(path string, info os.FileInfo, err error) error { 1375 if err != nil { 1376 return err 1377 } 1378 relPath, err := filepath.Rel(op.Project().GitHooks, path) 1379 if err != nil { 1380 return err 1381 } 1382 dst := filepath.Join(gitHooksDstDir, relPath) 1383 if info.IsDir() { 1384 return fmtError(os.MkdirAll(dst, 0755)) 1385 } 1386 src, err := ioutil.ReadFile(path) 1387 if err != nil { 1388 return fmtError(err) 1389 } 1390 // The file *must* be executable to be picked up by git. 1391 return fmtError(ioutil.WriteFile(dst, src, 0755)) 1392 } 1393 if err := filepath.Walk(op.Project().GitHooks, copyFn); err != nil { 1394 return err 1395 } 1396 } 1397 return nil 1398 }