github.com/btwiuse/jiri@v0.0.0-20191125065820-53353bcfef54/x.go (about) 1 // Copyright 2015 The Vanadium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Package jiri provides utilities used by the jiri tool and related tools. 6 package jiri 7 8 // TODO(toddw): Rename this package to github.com/btwiuse/jiri, and rename the tool itself to 9 // github.com/btwiuse/jiri/cmd/jiri 10 11 import ( 12 "encoding/xml" 13 "flag" 14 "fmt" 15 "io/ioutil" 16 "os" 17 "path/filepath" 18 "runtime" 19 "strconv" 20 "sync/atomic" 21 "time" 22 23 "github.com/btwiuse/jiri/analytics_util" 24 "github.com/btwiuse/jiri/cmdline" 25 "github.com/btwiuse/jiri/color" 26 "github.com/btwiuse/jiri/envvar" 27 "github.com/btwiuse/jiri/log" 28 "github.com/btwiuse/jiri/timing" 29 "github.com/btwiuse/jiri/tool" 30 ) 31 32 const ( 33 RootMetaDir = ".jiri_root" 34 ProjectMetaDir = ".git/jiri" 35 OldProjectMetaDir = ".jiri" 36 ConfigFile = "config" 37 DefaultCacheSubdir = "cache" 38 ProjectMetaFile = "metadata.v2" 39 ProjectConfigFile = "config" 40 JiriManifestFile = ".jiri_manifest" 41 42 // PreservePathEnv is the name of the environment variable that, when set to a 43 // non-empty value, causes jiri tools to use the existing PATH variable, 44 // rather than mutating it. 45 PreservePathEnv = "JIRI_PRESERVE_PATH" 46 ) 47 48 // Config represents jiri global config 49 type Config struct { 50 CachePath string `xml:"cache>path,omitempty"` 51 CipdParanoidMode string `xml:"cipd_paranoid_mode,omitempty"` 52 CipdMaxThreads int `xml:"cipd_max_threads,omitempty"` 53 Shared bool `xml:"cache>shared,omitempty"` 54 RewriteSsoToHttps bool `xml:"rewriteSsoToHttps,omitempty"` 55 SsoCookiePath string `xml:"SsoCookiePath,omitempty"` 56 LockfileEnabled string `xml:"lockfile>enabled,omitempty"` 57 LockfileName string `xml:"lockfile>name,omitempty"` 58 PrebuiltJSON string `xml:"prebuilt>JSON,omitempty"` 59 FetchingAttrs string `xml:"fetchingAttrs,omitempty"` 60 AnalyticsOptIn string `xml:"analytics>optin,omitempty"` 61 AnalyticsUserId string `xml:"analytics>userId,omitempty"` 62 Partial bool `xml:"partial,omitempty"` 63 // version user has opted-in to 64 AnalyticsVersion string `xml:"analytics>version,omitempty"` 65 KeepGitHooks bool `xml:"keepGitHooks,omitempty"` 66 67 XMLName struct{} `xml:"config"` 68 } 69 70 func (c *Config) Write(filename string) error { 71 if c.CachePath != "" { 72 var err error 73 c.CachePath, err = cleanPath(c.CachePath) 74 if err != nil { 75 return err 76 } 77 } 78 data, err := xml.MarshalIndent(c, "", " ") 79 if err != nil { 80 return err 81 } 82 return ioutil.WriteFile(filename, data, 0644) 83 } 84 85 func ConfigFromFile(filename string) (*Config, error) { 86 bytes, err := ioutil.ReadFile(filename) 87 if err != nil { 88 return nil, err 89 } 90 c := new(Config) 91 if err := xml.Unmarshal(bytes, c); err != nil { 92 return nil, err 93 } 94 return c, nil 95 } 96 97 // X holds the execution environment for the jiri tool and related tools. This 98 // includes the jiri filesystem root directory. 99 // 100 // TODO(toddw): Other jiri state should be transitioned to this struct, 101 // including the manifest and related operations. 102 type X struct { 103 *tool.Context 104 Root string 105 Usage func(format string, args ...interface{}) error 106 config *Config 107 Cache string 108 CipdParanoidMode bool 109 CipdMaxThreads int 110 Shared bool 111 Jobs uint 112 KeepGitHooks bool 113 RewriteSsoToHttps bool 114 LockfileEnabled bool 115 LockfileName string 116 SsoCookiePath string 117 Partial bool 118 PrebuiltJSON string 119 FetchingAttrs string 120 UsingSnapshot bool 121 UsingImportOverride bool 122 OverrideOptional bool 123 IgnoreLockConflicts bool 124 Color color.Color 125 Logger *log.Logger 126 failures uint32 127 Attempts uint 128 cleanupFuncs []func() 129 AnalyticsSession *analytics_util.AnalyticsSession 130 OverrideWarned bool 131 } 132 133 func (jirix *X) IncrementFailures() { 134 atomic.AddUint32(&jirix.failures, 1) 135 } 136 137 func (jirix *X) Failures() uint32 { 138 return atomic.LoadUint32(&jirix.failures) 139 } 140 141 // This is not thread safe 142 func (jirix *X) AddCleanupFunc(cleanup func()) { 143 jirix.cleanupFuncs = append(jirix.cleanupFuncs, cleanup) 144 } 145 146 // Executes all the cleanups added in LIFO order 147 func (jirix *X) RunCleanup() { 148 for _, fn := range jirix.cleanupFuncs { 149 // defer so that cleanups are executed in LIFO order 150 defer fn() 151 } 152 } 153 154 var ( 155 rootFlag string 156 jobsFlag uint 157 colorFlag string 158 quietVerboseFlag bool 159 debugVerboseFlag bool 160 traceVerboseFlag bool 161 showProgressFlag bool 162 progessWindowSizeFlag uint 163 timeLogThresholdFlag time.Duration 164 ) 165 166 // showRootFlag implements a flag that dumps the root dir and exits the 167 // program when it is set. 168 type showRootFlag struct{} 169 170 func (showRootFlag) IsBoolFlag() bool { return true } 171 func (showRootFlag) String() string { return "<just specify -show-root to activate>" } 172 func (showRootFlag) Set(string) error { 173 if root, err := findJiriRoot(nil); err != nil { 174 fmt.Printf("Error: %s\n", err) 175 os.Exit(1) 176 } else { 177 fmt.Println(root) 178 os.Exit(0) 179 } 180 return nil 181 } 182 183 var DefaultJobs = uint(runtime.NumCPU() * 2) 184 185 func init() { 186 // Cap jobs at 50 to avoid flooding Gerrit with too many requests 187 if DefaultJobs > 50 { 188 DefaultJobs = 50 189 } 190 flag.StringVar(&rootFlag, "root", "", "Jiri root directory") 191 flag.UintVar(&jobsFlag, "j", DefaultJobs, "Number of jobs (commands) to run simultaneously") 192 flag.StringVar(&colorFlag, "color", "auto", "Use color to format output. Values can be always, never and auto") 193 flag.BoolVar(&showProgressFlag, "show-progress", true, "Show progress.") 194 flag.Var(showRootFlag{}, "show-root", "Displays jiri root and exits.") 195 flag.UintVar(&progessWindowSizeFlag, "progress-window", 5, "Number of progress messages to show simultaneously. Should be between 1 and 10") 196 flag.DurationVar(&timeLogThresholdFlag, "time-log-threshold", time.Second*10, "Log time taken by operations if more than the passed value (eg 5s). This only works with -v and -vv.") 197 flag.BoolVar(&quietVerboseFlag, "quiet", false, "Only print user actionable messages.") 198 flag.BoolVar(&quietVerboseFlag, "q", false, "Same as -quiet") 199 flag.BoolVar(&debugVerboseFlag, "v", false, "Print debug level output.") 200 flag.BoolVar(&traceVerboseFlag, "vv", false, "Print trace level output.") 201 } 202 203 // NewX returns a new execution environment, given a cmdline env. 204 // It also prepends .jiri_root/bin to the PATH. 205 func NewX(env *cmdline.Env) (*X, error) { 206 cf := color.EnableColor(colorFlag) 207 if cf != color.ColorAuto && cf != color.ColorAlways && cf != color.ColorNever { 208 return nil, env.UsageErrorf("invalid value of -color flag") 209 } 210 color := color.NewColor(cf) 211 212 loggerLevel := log.InfoLevel 213 if quietVerboseFlag { 214 loggerLevel = log.WarningLevel 215 } else if traceVerboseFlag { 216 loggerLevel = log.TraceLevel 217 } else if debugVerboseFlag { 218 loggerLevel = log.DebugLevel 219 } 220 if progessWindowSizeFlag < 1 { 221 progessWindowSizeFlag = 1 222 } else if progessWindowSizeFlag > 10 { 223 progessWindowSizeFlag = 10 224 } 225 logger := log.NewLogger(loggerLevel, color, showProgressFlag, progessWindowSizeFlag, timeLogThresholdFlag, nil, nil) 226 227 ctx := tool.NewContextFromEnv(env) 228 root, err := findJiriRoot(ctx.Timer()) 229 if err != nil { 230 return nil, err 231 } 232 233 if jobsFlag == 0 { 234 return nil, fmt.Errorf("No of concurrent jobs should be more than zero") 235 } 236 237 x := &X{ 238 Context: ctx, 239 Root: root, 240 Usage: env.UsageErrorf, 241 Jobs: jobsFlag, 242 Color: color, 243 Logger: logger, 244 Attempts: 1, 245 } 246 configPath := filepath.Join(x.RootMetaDir(), ConfigFile) 247 if _, err := os.Stat(configPath); err == nil { 248 x.config, err = ConfigFromFile(configPath) 249 if err != nil { 250 return nil, err 251 } 252 } else if os.IsNotExist(err) { 253 x.config = &Config{} 254 } else { 255 return nil, err 256 } 257 if x.config != nil { 258 x.KeepGitHooks = x.config.KeepGitHooks 259 x.RewriteSsoToHttps = x.config.RewriteSsoToHttps 260 x.SsoCookiePath = x.config.SsoCookiePath 261 if x.config.LockfileEnabled == "" { 262 x.LockfileEnabled = true 263 } else { 264 if val, err := strconv.ParseBool(x.config.LockfileEnabled); err != nil { 265 return nil, fmt.Errorf("'config>lockfile>enable' flag should be true or false") 266 } else { 267 x.LockfileEnabled = val 268 } 269 } 270 if x.config.CipdParanoidMode == "" { 271 x.CipdParanoidMode = true 272 } else { 273 if val, err := strconv.ParseBool(x.config.CipdParanoidMode); err != nil { 274 return nil, fmt.Errorf("'config>cipd_paranoid_mode' flag should be true or false") 275 } else { 276 x.CipdParanoidMode = val 277 } 278 } 279 x.CipdMaxThreads = x.config.CipdMaxThreads 280 x.LockfileName = x.config.LockfileName 281 x.PrebuiltJSON = x.config.PrebuiltJSON 282 x.FetchingAttrs = x.config.FetchingAttrs 283 if x.LockfileName == "" { 284 x.LockfileName = "jiri.lock" 285 } 286 if x.PrebuiltJSON == "" { 287 x.PrebuiltJSON = "prebuilt.json" 288 } 289 } 290 x.Cache, err = findCache(root, x.config) 291 if x.config != nil { 292 x.Shared = x.config.Shared 293 x.Partial = x.config.Partial 294 } 295 296 if err != nil { 297 return nil, err 298 } 299 if ctx.Env()[PreservePathEnv] == "" { 300 // Prepend .jiri_root/bin to the PATH, so execing a binary will 301 // invoke the one in that directory, if it exists. This is crucial for jiri 302 // subcommands, where we want to invoke the binary that jiri installed, not 303 // whatever is in the user's PATH. 304 // 305 // Note that we must modify the actual os env variable with os.SetEnv and 306 // also the ctx.env, so that execing a binary through the os/exec package 307 // and with ctx.Run both have the correct behavior. 308 newPath := envvar.PrependUniqueToken(ctx.Env()["PATH"], string(os.PathListSeparator), x.BinDir()) 309 ctx.Env()["PATH"] = newPath 310 if err := os.Setenv("PATH", newPath); err != nil { 311 return nil, err 312 } 313 } 314 return x, nil 315 } 316 317 func cleanPath(path string) (string, error) { 318 result, err := filepath.EvalSymlinks(path) 319 if err != nil { 320 return "", fmt.Errorf("EvalSymlinks(%v) failed: %v", path, err) 321 } 322 if !filepath.IsAbs(result) { 323 return "", fmt.Errorf("%v isn't an absolute path", result) 324 } 325 return filepath.Clean(result), nil 326 } 327 328 func findCache(root string, config *Config) (string, error) { 329 // Use flag variable if set. 330 if config != nil && config.CachePath != "" { 331 return cleanPath(config.CachePath) 332 } 333 334 // Check default location under .jiri_root. 335 defaultCache := filepath.Join(root, DefaultCacheSubdir) 336 fi, err := os.Stat(defaultCache) 337 if err != nil { 338 if os.IsNotExist(err) { 339 return "", nil 340 } 341 return "", err 342 } 343 344 // .jiri_root/cache exists and is a directory (success). 345 if fi.IsDir() { 346 return defaultCache, nil 347 } 348 349 // defaultCache exists but is not a directory. Assume the user is 350 // up to something and there's no real cache directory. 351 return "", nil 352 } 353 354 func findJiriRoot(timer *timing.Timer) (string, error) { 355 if timer != nil { 356 timer.Push("find .jiri_root") 357 defer timer.Pop() 358 } 359 360 if rootFlag != "" { 361 return cleanPath(rootFlag) 362 } 363 364 wd, err := os.Getwd() 365 if err != nil { 366 return "", err 367 } 368 369 path, err := filepath.Abs(wd) 370 if err != nil { 371 return "", err 372 } 373 374 paths := []string{path} 375 for i := len(path) - 1; i >= 0; i-- { 376 if os.IsPathSeparator(path[i]) { 377 path = path[:i] 378 if path == "" { 379 path = "/" 380 } 381 paths = append(paths, path) 382 } 383 } 384 385 for _, path := range paths { 386 fi, err := os.Stat(filepath.Join(path, RootMetaDir)) 387 if err == nil && fi.IsDir() { 388 return path, nil 389 } 390 } 391 392 return "", fmt.Errorf("cannot find %v", RootMetaDir) 393 } 394 395 // FindRoot returns the root directory of the jiri environment. All state 396 // managed by jiri resides under this root. 397 // 398 // If the rootFlag variable is non-empty, we always attempt to use it. 399 // It must point to an absolute path, after symlinks are evaluated. 400 // 401 // Returns an empty string if the root directory cannot be determined, or if any 402 // errors are encountered. 403 // 404 // FindRoot should be rarely used; typically you should use NewX to create a new 405 // execution environment, and handle errors. An example of a valid usage is to 406 // initialize default flag values in an init func before main. 407 func FindRoot() string { 408 root, _ := findJiriRoot(nil) 409 return root 410 } 411 412 // Clone returns a clone of the environment. 413 func (x *X) Clone(opts tool.ContextOpts) *X { 414 return &X{ 415 Context: x.Context.Clone(opts), 416 Root: x.Root, 417 Usage: x.Usage, 418 Jobs: x.Jobs, 419 Cache: x.Cache, 420 Color: x.Color, 421 RewriteSsoToHttps: x.RewriteSsoToHttps, 422 Logger: x.Logger, 423 failures: x.failures, 424 Attempts: x.Attempts, 425 cleanupFuncs: x.cleanupFuncs, 426 AnalyticsSession: x.AnalyticsSession, 427 } 428 } 429 430 // UsageErrorf prints the error message represented by the printf-style format 431 // and args, followed by the usage output. The implementation typically calls 432 // cmdline.Env.UsageErrorf. 433 func (x *X) UsageErrorf(format string, args ...interface{}) error { 434 if x.Usage != nil { 435 return x.Usage(format, args...) 436 } 437 return fmt.Errorf(format, args...) 438 } 439 440 // RootMetaDir returns the path to the root metadata directory. 441 func (x *X) RootMetaDir() string { 442 return filepath.Join(x.Root, RootMetaDir) 443 } 444 445 // CIPDPath returns the path to directory containing cipd. 446 func (x *X) CIPDPath() string { 447 return filepath.Join(x.RootMetaDir(), "bin", "cipd") 448 } 449 450 // JiriManifestFile returns the path to the .jiri_manifest file. 451 func (x *X) JiriManifestFile() string { 452 return filepath.Join(x.Root, JiriManifestFile) 453 } 454 455 // BinDir returns the path to the bin directory. 456 func (x *X) BinDir() string { 457 return filepath.Join(x.RootMetaDir(), "bin") 458 } 459 460 // ScriptsDir returns the path to the scripts directory. 461 func (x *X) ScriptsDir() string { 462 return filepath.Join(x.RootMetaDir(), "scripts") 463 } 464 465 // UpdateHistoryDir returns the path to the update history directory. 466 func (x *X) UpdateHistoryDir() string { 467 return filepath.Join(x.RootMetaDir(), "update_history") 468 } 469 470 // UpdateHistoryLatestLink returns the path to a symlink that points to the 471 // latest update in the update history directory. 472 func (x *X) UpdateHistoryLatestLink() string { 473 return filepath.Join(x.UpdateHistoryDir(), "latest") 474 } 475 476 // UpdateHistorySecondLatestLink returns the path to a symlink that points to 477 // the second latest update in the update history directory. 478 func (x *X) UpdateHistorySecondLatestLink() string { 479 return filepath.Join(x.UpdateHistoryDir(), "second-latest") 480 } 481 482 // UpdateHistoryLogDir returns the path to the update history directory. 483 func (x *X) UpdateHistoryLogDir() string { 484 return filepath.Join(x.RootMetaDir(), "update_history_log") 485 } 486 487 // UpdateHistoryLogLatestLink returns the path to a symlink that points to the 488 // latest update in the update history directory. 489 func (x *X) UpdateHistoryLogLatestLink() string { 490 return filepath.Join(x.UpdateHistoryLogDir(), "latest") 491 } 492 493 // UpdateHistoryLogSecondLatestLink returns the path to a symlink that points to 494 // the second latest update in the update history directory. 495 func (x *X) UpdateHistoryLogSecondLatestLink() string { 496 return filepath.Join(x.UpdateHistoryLogDir(), "second-latest") 497 } 498 499 // RunnerFunc is an adapter that turns regular functions into cmdline.Runner. 500 // This is similar to cmdline.RunnerFunc, but the first function argument is 501 // jiri.X, rather than cmdline.Env. 502 func RunnerFunc(run func(*X, []string) error) cmdline.Runner { 503 return runner(run) 504 } 505 506 type runner func(*X, []string) error 507 508 func (r runner) Run(env *cmdline.Env, args []string) error { 509 x, err := NewX(env) 510 if err != nil { 511 return err 512 } 513 defer x.RunCleanup() 514 enabledAnalytics := false 515 userId := "" 516 analyticsCommandMsg := fmt.Sprintf("To check what data we collect run: %s\n"+ 517 "To opt-in run: %s\n"+ 518 "To opt-out run: %s", 519 x.Color.Yellow("jiri init -show-analytics-data"), 520 x.Color.Yellow("jiri init -analytics-opt=true %q", x.Root), 521 x.Color.Yellow("jiri init -analytics-opt=false %q", x.Root)) 522 if x.config == nil || x.config.AnalyticsOptIn == "" { 523 x.Logger.Warningf("Please opt in or out of analytics collection. You will receive this warning until an option is selected.\n%s\n\n", analyticsCommandMsg) 524 } else if x.config.AnalyticsOptIn == "yes" { 525 if x.config.AnalyticsUserId == "" || x.config.AnalyticsVersion == "" { 526 x.Logger.Warningf("Please opt in or out of analytics collection. You will receive this warning until an option is selected.\n%s\n\n", analyticsCommandMsg) 527 } else if x.config.AnalyticsVersion != analytics_util.Version { 528 x.Logger.Warningf("You have opted in for old version of data collection. Please opt in/out again\n%s\n\n", analyticsCommandMsg) 529 } else { 530 userId = x.config.AnalyticsUserId 531 enabledAnalytics = true 532 } 533 } 534 as := analytics_util.NewAnalyticsSession(enabledAnalytics, "UA-101128147-1", userId) 535 x.AnalyticsSession = as 536 id := as.AddCommand(env.CommandName, env.CommandFlags) 537 538 err = r(x, args) 539 x.Logger.DisableProgress() 540 541 as.Done(id) 542 as.SendAllAndWaitToFinish() 543 return err 544 }