github.com/mckael/restic@v0.8.3/cmd/restic/global.go (about) 1 package main 2 3 import ( 4 "context" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "os" 9 "path/filepath" 10 "runtime" 11 "strings" 12 "syscall" 13 "time" 14 15 "github.com/restic/restic/internal/backend" 16 "github.com/restic/restic/internal/backend/azure" 17 "github.com/restic/restic/internal/backend/b2" 18 "github.com/restic/restic/internal/backend/gs" 19 "github.com/restic/restic/internal/backend/local" 20 "github.com/restic/restic/internal/backend/location" 21 "github.com/restic/restic/internal/backend/rest" 22 "github.com/restic/restic/internal/backend/s3" 23 "github.com/restic/restic/internal/backend/sftp" 24 "github.com/restic/restic/internal/backend/swift" 25 "github.com/restic/restic/internal/cache" 26 "github.com/restic/restic/internal/debug" 27 "github.com/restic/restic/internal/fs" 28 "github.com/restic/restic/internal/limiter" 29 "github.com/restic/restic/internal/options" 30 "github.com/restic/restic/internal/repository" 31 "github.com/restic/restic/internal/restic" 32 33 "github.com/restic/restic/internal/errors" 34 35 "golang.org/x/crypto/ssh/terminal" 36 ) 37 38 var version = "compiled manually" 39 40 // GlobalOptions hold all global options for restic. 41 type GlobalOptions struct { 42 Repo string 43 PasswordFile string 44 Quiet bool 45 NoLock bool 46 JSON bool 47 CacheDir string 48 NoCache bool 49 CACerts []string 50 TLSClientCert string 51 CleanupCache bool 52 53 LimitUploadKb int 54 LimitDownloadKb int 55 56 ctx context.Context 57 password string 58 stdout io.Writer 59 stderr io.Writer 60 61 Options []string 62 63 extended options.Options 64 } 65 66 var globalOptions = GlobalOptions{ 67 stdout: os.Stdout, 68 stderr: os.Stderr, 69 } 70 71 func init() { 72 var cancel context.CancelFunc 73 globalOptions.ctx, cancel = context.WithCancel(context.Background()) 74 AddCleanupHandler(func() error { 75 cancel() 76 return nil 77 }) 78 79 f := cmdRoot.PersistentFlags() 80 f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "repository to backup to or restore from (default: $RESTIC_REPOSITORY)") 81 f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", os.Getenv("RESTIC_PASSWORD_FILE"), "read the repository password from a file (default: $RESTIC_PASSWORD_FILE)") 82 f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report") 83 f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repo, this allows some operations on read-only repos") 84 f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it") 85 f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache directory") 86 f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache") 87 f.StringSliceVar(&globalOptions.CACerts, "cacert", nil, "path to load root certificates from (default: use system certificates)") 88 f.StringVar(&globalOptions.TLSClientCert, "tls-client-cert", "", "path to a file containing PEM encoded TLS client certificate and private key") 89 f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") 90 f.IntVar(&globalOptions.LimitUploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)") 91 f.IntVar(&globalOptions.LimitDownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)") 92 f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)") 93 94 restoreTerminal() 95 } 96 97 // checkErrno returns nil when err is set to syscall.Errno(0), since this is no 98 // error condition. 99 func checkErrno(err error) error { 100 e, ok := err.(syscall.Errno) 101 if !ok { 102 return err 103 } 104 105 if e == 0 { 106 return nil 107 } 108 109 return err 110 } 111 112 func stdinIsTerminal() bool { 113 return terminal.IsTerminal(int(os.Stdin.Fd())) 114 } 115 116 func stdoutIsTerminal() bool { 117 return terminal.IsTerminal(int(os.Stdout.Fd())) 118 } 119 120 func stdoutTerminalWidth() int { 121 w, _, err := terminal.GetSize(int(os.Stdout.Fd())) 122 if err != nil { 123 return 0 124 } 125 return w 126 } 127 128 // restoreTerminal installs a cleanup handler that restores the previous 129 // terminal state on exit. 130 func restoreTerminal() { 131 if !stdoutIsTerminal() { 132 return 133 } 134 135 fd := int(os.Stdout.Fd()) 136 state, err := terminal.GetState(fd) 137 if err != nil { 138 fmt.Fprintf(os.Stderr, "unable to get terminal state: %v\n", err) 139 return 140 } 141 142 AddCleanupHandler(func() error { 143 err := checkErrno(terminal.Restore(fd, state)) 144 if err != nil { 145 fmt.Fprintf(os.Stderr, "unable to get restore terminal state: %#+v\n", err) 146 } 147 return err 148 }) 149 } 150 151 // ClearLine creates a platform dependent string to clear the current 152 // line, so it can be overwritten. ANSI sequences are not supported on 153 // current windows cmd shell. 154 func ClearLine() string { 155 if runtime.GOOS == "windows" { 156 if w := stdoutTerminalWidth(); w > 0 { 157 return strings.Repeat(" ", w-1) + "\r" 158 } 159 return "" 160 } 161 return "\x1b[2K" 162 } 163 164 // Printf writes the message to the configured stdout stream. 165 func Printf(format string, args ...interface{}) { 166 _, err := fmt.Fprintf(globalOptions.stdout, format, args...) 167 if err != nil { 168 fmt.Fprintf(os.Stderr, "unable to write to stdout: %v\n", err) 169 Exit(100) 170 } 171 } 172 173 // Verbosef calls Printf to write the message when the verbose flag is set. 174 func Verbosef(format string, args ...interface{}) { 175 if globalOptions.Quiet { 176 return 177 } 178 179 Printf(format, args...) 180 } 181 182 // PrintProgress wraps fmt.Printf to handle the difference in writing progress 183 // information to terminals and non-terminal stdout 184 func PrintProgress(format string, args ...interface{}) { 185 var ( 186 message string 187 carriageControl string 188 ) 189 message = fmt.Sprintf(format, args...) 190 191 if !(strings.HasSuffix(message, "\r") || strings.HasSuffix(message, "\n")) { 192 if stdoutIsTerminal() { 193 carriageControl = "\r" 194 } else { 195 carriageControl = "\n" 196 } 197 message = fmt.Sprintf("%s%s", message, carriageControl) 198 } 199 200 if stdoutIsTerminal() { 201 message = fmt.Sprintf("%s%s", ClearLine(), message) 202 } 203 204 fmt.Print(message) 205 } 206 207 // Warnf writes the message to the configured stderr stream. 208 func Warnf(format string, args ...interface{}) { 209 _, err := fmt.Fprintf(globalOptions.stderr, format, args...) 210 if err != nil { 211 fmt.Fprintf(os.Stderr, "unable to write to stderr: %v\n", err) 212 Exit(100) 213 } 214 } 215 216 // Exitf uses Warnf to write the message and then terminates the process with 217 // the given exit code. 218 func Exitf(exitcode int, format string, args ...interface{}) { 219 if format[len(format)-1] != '\n' { 220 format += "\n" 221 } 222 223 Warnf(format, args...) 224 Exit(exitcode) 225 } 226 227 // resolvePassword determines the password to be used for opening the repository. 228 func resolvePassword(opts GlobalOptions, env string) (string, error) { 229 if opts.PasswordFile != "" { 230 s, err := ioutil.ReadFile(opts.PasswordFile) 231 if os.IsNotExist(err) { 232 return "", errors.Fatalf("%s does not exist", opts.PasswordFile) 233 } 234 return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile") 235 } 236 237 if pwd := os.Getenv(env); pwd != "" { 238 return pwd, nil 239 } 240 241 return "", nil 242 } 243 244 // readPassword reads the password from the given reader directly. 245 func readPassword(in io.Reader) (password string, err error) { 246 buf := make([]byte, 1000) 247 n, err := io.ReadFull(in, buf) 248 buf = buf[:n] 249 250 if err != nil && errors.Cause(err) != io.ErrUnexpectedEOF { 251 return "", errors.Wrap(err, "ReadFull") 252 } 253 254 return strings.TrimRight(string(buf), "\r\n"), nil 255 } 256 257 // readPasswordTerminal reads the password from the given reader which must be a 258 // tty. Prompt is printed on the writer out before attempting to read the 259 // password. 260 func readPasswordTerminal(in *os.File, out io.Writer, prompt string) (password string, err error) { 261 fmt.Fprint(out, prompt) 262 buf, err := terminal.ReadPassword(int(in.Fd())) 263 fmt.Fprintln(out) 264 if err != nil { 265 return "", errors.Wrap(err, "ReadPassword") 266 } 267 268 password = string(buf) 269 return password, nil 270 } 271 272 // ReadPassword reads the password from a password file, the environment 273 // variable RESTIC_PASSWORD or prompts the user. 274 func ReadPassword(opts GlobalOptions, prompt string) (string, error) { 275 if opts.password != "" { 276 return opts.password, nil 277 } 278 279 var ( 280 password string 281 err error 282 ) 283 284 if stdinIsTerminal() { 285 password, err = readPasswordTerminal(os.Stdin, os.Stderr, prompt) 286 } else { 287 password, err = readPassword(os.Stdin) 288 } 289 290 if err != nil { 291 return "", errors.Wrap(err, "unable to read password") 292 } 293 294 if len(password) == 0 { 295 return "", errors.Fatal("an empty password is not a password") 296 } 297 298 return password, nil 299 } 300 301 // ReadPasswordTwice calls ReadPassword two times and returns an error when the 302 // passwords don't match. 303 func ReadPasswordTwice(gopts GlobalOptions, prompt1, prompt2 string) (string, error) { 304 pw1, err := ReadPassword(gopts, prompt1) 305 if err != nil { 306 return "", err 307 } 308 pw2, err := ReadPassword(gopts, prompt2) 309 if err != nil { 310 return "", err 311 } 312 313 if pw1 != pw2 { 314 return "", errors.Fatal("passwords do not match") 315 } 316 317 return pw1, nil 318 } 319 320 const maxKeys = 20 321 322 // OpenRepository reads the password and opens the repository. 323 func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { 324 if opts.Repo == "" { 325 return nil, errors.Fatal("Please specify repository location (-r)") 326 } 327 328 be, err := open(opts.Repo, opts, opts.extended) 329 if err != nil { 330 return nil, err 331 } 332 333 be = backend.NewRetryBackend(be, 10, func(msg string, err error, d time.Duration) { 334 Warnf("%v returned error, retrying after %v: %v\n", msg, d, err) 335 }) 336 337 s := repository.New(be) 338 339 opts.password, err = ReadPassword(opts, "enter password for repository: ") 340 if err != nil { 341 return nil, err 342 } 343 344 err = s.SearchKey(opts.ctx, opts.password, maxKeys) 345 if err != nil { 346 return nil, err 347 } 348 349 if stdoutIsTerminal() { 350 Verbosef("password is correct\n") 351 } 352 353 if opts.NoCache { 354 return s, nil 355 } 356 357 c, err := cache.New(s.Config().ID, opts.CacheDir) 358 if err != nil { 359 Warnf("unable to open cache: %v\n", err) 360 return s, nil 361 } 362 363 // start using the cache 364 s.UseCache(c) 365 366 oldCacheDirs, err := cache.Old(c.Base) 367 if err != nil { 368 Warnf("unable to find old cache directories: %v", err) 369 } 370 371 // nothing more to do if no old cache dirs could be found 372 if len(oldCacheDirs) == 0 { 373 return s, nil 374 } 375 376 // cleanup old cache dirs if instructed to do so 377 if opts.CleanupCache { 378 Printf("removing %d old cache dirs from %v\n", len(oldCacheDirs), c.Base) 379 380 for _, item := range oldCacheDirs { 381 dir := filepath.Join(c.Base, item) 382 err = fs.RemoveAll(dir) 383 if err != nil { 384 Warnf("unable to remove %v: %v\n", dir, err) 385 } 386 } 387 } else { 388 if stdoutIsTerminal() { 389 Verbosef("found %d old cache directories in %v, pass --cleanup-cache to remove them\n", 390 len(oldCacheDirs), c.Base) 391 } 392 } 393 394 return s, nil 395 } 396 397 func parseConfig(loc location.Location, opts options.Options) (interface{}, error) { 398 // only apply options for a particular backend here 399 opts = opts.Extract(loc.Scheme) 400 401 switch loc.Scheme { 402 case "local": 403 cfg := loc.Config.(local.Config) 404 if err := opts.Apply(loc.Scheme, &cfg); err != nil { 405 return nil, err 406 } 407 408 debug.Log("opening local repository at %#v", cfg) 409 return cfg, nil 410 411 case "sftp": 412 cfg := loc.Config.(sftp.Config) 413 if err := opts.Apply(loc.Scheme, &cfg); err != nil { 414 return nil, err 415 } 416 417 debug.Log("opening sftp repository at %#v", cfg) 418 return cfg, nil 419 420 case "s3": 421 cfg := loc.Config.(s3.Config) 422 if cfg.KeyID == "" { 423 cfg.KeyID = os.Getenv("AWS_ACCESS_KEY_ID") 424 } 425 426 if cfg.Secret == "" { 427 cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY") 428 } 429 430 if err := opts.Apply(loc.Scheme, &cfg); err != nil { 431 return nil, err 432 } 433 434 debug.Log("opening s3 repository at %#v", cfg) 435 return cfg, nil 436 437 case "gs": 438 cfg := loc.Config.(gs.Config) 439 if cfg.ProjectID == "" { 440 cfg.ProjectID = os.Getenv("GOOGLE_PROJECT_ID") 441 } 442 443 if cfg.JSONKeyPath == "" { 444 if path := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"); path != "" { 445 // Check read access 446 if _, err := ioutil.ReadFile(path); err != nil { 447 return nil, errors.Fatalf("Failed to read google credential from file %v: %v", path, err) 448 } 449 cfg.JSONKeyPath = path 450 } else { 451 return nil, errors.Fatal("No credential file path is set") 452 } 453 } 454 455 if err := opts.Apply(loc.Scheme, &cfg); err != nil { 456 return nil, err 457 } 458 459 debug.Log("opening gs repository at %#v", cfg) 460 return cfg, nil 461 462 case "azure": 463 cfg := loc.Config.(azure.Config) 464 if cfg.AccountName == "" { 465 cfg.AccountName = os.Getenv("AZURE_ACCOUNT_NAME") 466 } 467 468 if cfg.AccountKey == "" { 469 cfg.AccountKey = os.Getenv("AZURE_ACCOUNT_KEY") 470 } 471 472 if err := opts.Apply(loc.Scheme, &cfg); err != nil { 473 return nil, err 474 } 475 476 debug.Log("opening gs repository at %#v", cfg) 477 return cfg, nil 478 479 case "swift": 480 cfg := loc.Config.(swift.Config) 481 482 if err := swift.ApplyEnvironment("", &cfg); err != nil { 483 return nil, err 484 } 485 486 if err := opts.Apply(loc.Scheme, &cfg); err != nil { 487 return nil, err 488 } 489 490 debug.Log("opening swift repository at %#v", cfg) 491 return cfg, nil 492 493 case "b2": 494 cfg := loc.Config.(b2.Config) 495 496 if cfg.AccountID == "" { 497 cfg.AccountID = os.Getenv("B2_ACCOUNT_ID") 498 } 499 500 if cfg.AccountID == "" { 501 return nil, errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty") 502 } 503 504 if cfg.Key == "" { 505 cfg.Key = os.Getenv("B2_ACCOUNT_KEY") 506 } 507 508 if cfg.Key == "" { 509 return nil, errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty") 510 } 511 512 if err := opts.Apply(loc.Scheme, &cfg); err != nil { 513 return nil, err 514 } 515 516 debug.Log("opening b2 repository at %#v", cfg) 517 return cfg, nil 518 case "rest": 519 cfg := loc.Config.(rest.Config) 520 if err := opts.Apply(loc.Scheme, &cfg); err != nil { 521 return nil, err 522 } 523 524 debug.Log("opening rest repository at %#v", cfg) 525 return cfg, nil 526 } 527 528 return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) 529 } 530 531 // Open the backend specified by a location config. 532 func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { 533 debug.Log("parsing location %v", s) 534 loc, err := location.Parse(s) 535 if err != nil { 536 return nil, errors.Fatalf("parsing repository location failed: %v", err) 537 } 538 539 var be restic.Backend 540 541 cfg, err := parseConfig(loc, opts) 542 if err != nil { 543 return nil, err 544 } 545 546 tropts := backend.TransportOptions{ 547 RootCertFilenames: globalOptions.CACerts, 548 TLSClientCertKeyFilename: globalOptions.TLSClientCert, 549 } 550 rt, err := backend.Transport(tropts) 551 if err != nil { 552 return nil, err 553 } 554 555 // wrap the transport so that the throughput via HTTP is limited 556 rt = limiter.NewStaticLimiter(gopts.LimitUploadKb, gopts.LimitDownloadKb).Transport(rt) 557 558 switch loc.Scheme { 559 case "local": 560 be, err = local.Open(cfg.(local.Config)) 561 // wrap the backend in a LimitBackend so that the throughput is limited 562 be = limiter.LimitBackend(be, limiter.NewStaticLimiter(gopts.LimitUploadKb, gopts.LimitDownloadKb)) 563 case "sftp": 564 be, err = sftp.Open(cfg.(sftp.Config)) 565 // wrap the backend in a LimitBackend so that the throughput is limited 566 be = limiter.LimitBackend(be, limiter.NewStaticLimiter(gopts.LimitUploadKb, gopts.LimitDownloadKb)) 567 case "s3": 568 be, err = s3.Open(cfg.(s3.Config), rt) 569 case "gs": 570 be, err = gs.Open(cfg.(gs.Config), rt) 571 case "azure": 572 be, err = azure.Open(cfg.(azure.Config), rt) 573 case "swift": 574 be, err = swift.Open(cfg.(swift.Config), rt) 575 case "b2": 576 be, err = b2.Open(globalOptions.ctx, cfg.(b2.Config), rt) 577 case "rest": 578 be, err = rest.Open(cfg.(rest.Config), rt) 579 580 default: 581 return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) 582 } 583 584 if err != nil { 585 return nil, errors.Fatalf("unable to open repo at %v: %v", s, err) 586 } 587 588 // check if config is there 589 fi, err := be.Stat(globalOptions.ctx, restic.Handle{Type: restic.ConfigFile}) 590 if err != nil { 591 return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, s) 592 } 593 594 if fi.Size == 0 { 595 return nil, errors.New("config file has zero size, invalid repository?") 596 } 597 598 return be, nil 599 } 600 601 // Create the backend specified by URI. 602 func create(s string, opts options.Options) (restic.Backend, error) { 603 debug.Log("parsing location %v", s) 604 loc, err := location.Parse(s) 605 if err != nil { 606 return nil, err 607 } 608 609 cfg, err := parseConfig(loc, opts) 610 if err != nil { 611 return nil, err 612 } 613 614 tropts := backend.TransportOptions{ 615 RootCertFilenames: globalOptions.CACerts, 616 TLSClientCertKeyFilename: globalOptions.TLSClientCert, 617 } 618 rt, err := backend.Transport(tropts) 619 if err != nil { 620 return nil, err 621 } 622 623 switch loc.Scheme { 624 case "local": 625 return local.Create(cfg.(local.Config)) 626 case "sftp": 627 return sftp.Create(cfg.(sftp.Config)) 628 case "s3": 629 return s3.Create(cfg.(s3.Config), rt) 630 case "gs": 631 return gs.Create(cfg.(gs.Config), rt) 632 case "azure": 633 return azure.Create(cfg.(azure.Config), rt) 634 case "swift": 635 return swift.Open(cfg.(swift.Config), rt) 636 case "b2": 637 return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt) 638 case "rest": 639 return rest.Create(cfg.(rest.Config), rt) 640 } 641 642 debug.Log("invalid repository scheme: %v", s) 643 return nil, errors.Fatalf("invalid scheme %q", loc.Scheme) 644 }