github.com/btwiuse/jiri@v0.0.0-20191125065820-53353bcfef54/project/source_manifest.go (about) 1 // Copyright 2017 The Fuchsia Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package project 6 7 import ( 8 "encoding/json" 9 "fmt" 10 "io/ioutil" 11 "net/url" 12 "os" 13 "path/filepath" 14 "strings" 15 "sync" 16 17 "github.com/btwiuse/jiri" 18 "github.com/btwiuse/jiri/gerrit" 19 "github.com/btwiuse/jiri/gitutil" 20 ) 21 22 const ( 23 SourceManifestVersion = int32(0) 24 ) 25 26 // This was created using proto file: https://github.com/luci/recipes-py/blob/master/recipe_engine/source_manifest.proto. 27 type SourceManifest_GitCheckout struct { 28 // The canonicalized URL of the original repo that is considered the “source 29 // of truth” for the source code. Ex. 30 // https://chromium.googlesource.com/chromium/tools/build.git 31 // https://github.com/luci/recipes-py 32 RepoUrl string `json:"repo_url,omitempty"` 33 34 // If different from repo_url, this can be the URL of the repo that the source 35 // was actually fetched from (i.e. a mirror). Ex. 36 // https://chromium.googlesource.com/external/github.com/luci/recipes-py 37 // 38 // If this is empty, it's presumed to be equal to repo_url. 39 FetchUrl string `json:"fetch_url,omitempty"` 40 41 // The fully resolved revision (commit hash) of the source. Ex. 42 // 3617b0eea7ec74b8e731a23fed2f4070cbc284c4 43 // 44 // Note that this is the raw revision bytes, not their hex-encoded form. 45 Revision string `json:"revision,omitempty"` 46 47 // The ref that the task used to resolve/fetch the revision of the source 48 // (if any). Ex. 49 // refs/heads/master 50 // refs/changes/04/511804/4 51 // 52 // This should always be a ref on the hosted repo (not any local alias 53 // like 'refs/remotes/...'). 54 // 55 // This should always be an absolute ref (i.e. starts with 'refs/'). An 56 // example of a non-absolute ref would be 'master'. 57 FetchRef string `json:"fetch_ref,omitempty"` 58 } 59 60 type SourceManifest_Directory struct { 61 GitCheckout *SourceManifest_GitCheckout `json:"git_checkout,omitempty"` 62 } 63 64 type SourceManifest struct { 65 // Version will increment on backwards-incompatible changes only. Backwards 66 // compatible changes will not alter this version number. 67 // 68 // Currently, the only valid version number is 0. 69 Version int32 `json:"version"` 70 71 // Map of local file system directory path (with forward slashes) to 72 // a Directory message containing one or more deployments. 73 // 74 // The local path is relative to some job-specific root. This should be used 75 // for informational/display/organization purposes, and should not be used as 76 // a global primary key. i.e. if you depend on chromium/src.git being in 77 // a folder called “src”, I will find you and make really angry faces at you 78 // until you change it...(╬ಠ益ಠ). Instead, implementations should consider 79 // indexing by e.g. git repository URL or cipd package name as more better 80 // primary keys. 81 Directories map[string]*SourceManifest_Directory `json:"directories"` 82 } 83 84 func getCLRefByCommit(jirix *jiri.X, gerritHost, revision string) (string, error) { 85 hostUrl, err := url.Parse(gerritHost) 86 if err != nil { 87 return "", fmt.Errorf("invalid gerrit host %q: %s", gerritHost, err) 88 } 89 g := gerrit.New(jirix, hostUrl) 90 cls, err := g.ListChangesByCommit(revision) 91 if err != nil { 92 return "", fmt.Errorf("not able to get CL for revision %s: %s", revision, err) 93 } 94 for _, c := range cls { 95 if v, ok := c.Revisions[revision]; ok { 96 return v.Fetch.Ref, nil 97 } 98 } 99 return "", nil 100 } 101 102 func NewSourceManifest(jirix *jiri.X, projects Projects) (*SourceManifest, MultiError) { 103 jirix.TimerPush("create source manifest") 104 defer jirix.TimerPop() 105 106 workQueue := make(chan Project, len(projects)) 107 for _, proj := range projects { 108 if err := proj.relativizePaths(jirix.Root); err != nil { 109 return nil, MultiError{err} 110 } 111 workQueue <- proj 112 } 113 close(workQueue) 114 errs := make(chan error, len(projects)) 115 sm := &SourceManifest{ 116 Version: SourceManifestVersion, 117 Directories: make(map[string]*SourceManifest_Directory), 118 } 119 var mux sync.Mutex 120 processProject := func(proj Project) error { 121 gc := &SourceManifest_GitCheckout{ 122 RepoUrl: rewriteRemote(jirix, proj.Remote), 123 } 124 scm := gitutil.New(jirix, gitutil.RootDirOpt(filepath.Join(jirix.Root, proj.Path))) 125 if rev, err := scm.CurrentRevision(); err != nil { 126 return err 127 } else { 128 gc.Revision = rev 129 } 130 if proj.RemoteBranch == "" { 131 proj.RemoteBranch = "master" 132 } 133 branchMap, err := scm.ListRemoteBranchesContainingRef(gc.Revision) 134 if err != nil { 135 return err 136 } 137 if branchMap["origin/"+proj.RemoteBranch] { 138 gc.FetchRef = "refs/heads/" + proj.RemoteBranch 139 } else { 140 for b, _ := range branchMap { 141 if strings.HasPrefix(b, "origin/HEAD ") { 142 continue 143 } 144 if strings.HasPrefix(b, "origin") { 145 gc.FetchRef = "refs/heads/" + strings.TrimLeft(b, "origin/") 146 break 147 } 148 } 149 150 // Try getting from gerrit 151 if gc.FetchRef == "" && proj.GerritHost != "" { 152 if ref, err := getCLRefByCommit(jirix, proj.GerritHost, gc.Revision); err != nil { 153 // Don't fail 154 jirix.Logger.Debugf("Error while fetching from gerrit for project %q: %s", proj.Name, err) 155 } else if ref == "" { 156 jirix.Logger.Debugf("Cannot get ref for project: %q, revision: %q", proj.Name, gc.Revision) 157 } else { 158 gc.FetchRef = ref 159 } 160 } 161 } 162 mux.Lock() 163 sm.Directories[proj.Path] = &SourceManifest_Directory{GitCheckout: gc} 164 mux.Unlock() 165 return nil 166 } 167 168 var wg sync.WaitGroup 169 for i := uint(0); i < jirix.Jobs; i++ { 170 wg.Add(1) 171 go func() { 172 defer wg.Done() 173 for p := range workQueue { 174 if err := processProject(p); err != nil { 175 errs <- err 176 } 177 } 178 }() 179 } 180 wg.Wait() 181 close(errs) 182 var multiErr MultiError 183 for err := range errs { 184 multiErr = append(multiErr, err) 185 } 186 return sm, multiErr 187 } 188 189 func (sm *SourceManifest) ToFile(jirix *jiri.X, filename string) error { 190 if err := os.MkdirAll(filepath.Dir(filename), 0755); err != nil { 191 return fmtError(err) 192 } 193 out, err := json.MarshalIndent(sm, "", " ") 194 if err != nil { 195 return fmt.Errorf("failed to serialize JSON output: %s\n", err) 196 } 197 198 err = ioutil.WriteFile(filename, out, 0600) 199 if err != nil { 200 return fmt.Errorf("failed write JSON output to %s: %s\n", filename, err) 201 } 202 203 return nil 204 }