github.com/mckael/restic@v0.8.3/cmd/restic/cmd_backup.go (about) 1 package main 2 3 import ( 4 "bufio" 5 "fmt" 6 "io" 7 "os" 8 "path" 9 "path/filepath" 10 "strings" 11 "time" 12 13 "github.com/spf13/cobra" 14 15 "github.com/restic/restic/internal/archiver" 16 "github.com/restic/restic/internal/debug" 17 "github.com/restic/restic/internal/errors" 18 "github.com/restic/restic/internal/fs" 19 "github.com/restic/restic/internal/restic" 20 ) 21 22 var cmdBackup = &cobra.Command{ 23 Use: "backup [flags] FILE/DIR [FILE/DIR] ...", 24 Short: "Create a new backup of files and/or directories", 25 Long: ` 26 The "backup" command creates a new snapshot and saves the files and directories 27 given as the arguments. 28 `, 29 PreRun: func(cmd *cobra.Command, args []string) { 30 if backupOptions.Hostname == "" { 31 hostname, err := os.Hostname() 32 if err != nil { 33 debug.Log("os.Hostname() returned err: %v", err) 34 return 35 } 36 backupOptions.Hostname = hostname 37 } 38 }, 39 DisableAutoGenTag: true, 40 RunE: func(cmd *cobra.Command, args []string) error { 41 if backupOptions.Stdin && backupOptions.FilesFrom == "-" { 42 return errors.Fatal("cannot use both `--stdin` and `--files-from -`") 43 } 44 45 if backupOptions.Stdin { 46 return readBackupFromStdin(backupOptions, globalOptions, args) 47 } 48 49 return runBackup(backupOptions, globalOptions, args) 50 }, 51 } 52 53 // BackupOptions bundles all options for the backup command. 54 type BackupOptions struct { 55 Parent string 56 Force bool 57 Excludes []string 58 ExcludeFiles []string 59 ExcludeOtherFS bool 60 ExcludeIfPresent []string 61 ExcludeCaches bool 62 Stdin bool 63 StdinFilename string 64 Tags []string 65 Hostname string 66 FilesFrom string 67 TimeStamp string 68 WithAtime bool 69 } 70 71 var backupOptions BackupOptions 72 73 func init() { 74 cmdRoot.AddCommand(cmdBackup) 75 76 f := cmdBackup.Flags() 77 f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)") 78 f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`) 79 f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") 80 f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)") 81 f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems") 82 f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes filename[:header], exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)") 83 f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file`) 84 f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin") 85 f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin") 86 f.StringArrayVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)") 87 f.StringVar(&backupOptions.Hostname, "hostname", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag") 88 f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)") 89 f.StringVar(&backupOptions.TimeStamp, "time", "", "time of the backup (ex. '2012-11-01 22:08:41') (default: now)") 90 f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories") 91 } 92 93 func newScanProgress(gopts GlobalOptions) *restic.Progress { 94 if gopts.Quiet { 95 return nil 96 } 97 98 p := restic.NewProgress() 99 p.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) { 100 if IsProcessBackground() { 101 return 102 } 103 104 PrintProgress("[%s] %d directories, %d files, %s", formatDuration(d), s.Dirs, s.Files, formatBytes(s.Bytes)) 105 } 106 107 p.OnDone = func(s restic.Stat, d time.Duration, ticker bool) { 108 PrintProgress("scanned %d directories, %d files in %s\n", s.Dirs, s.Files, formatDuration(d)) 109 } 110 111 return p 112 } 113 114 func newArchiveProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress { 115 if gopts.Quiet { 116 return nil 117 } 118 119 archiveProgress := restic.NewProgress() 120 121 var bps, eta uint64 122 itemsTodo := todo.Files + todo.Dirs 123 124 archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) { 125 if IsProcessBackground() { 126 return 127 } 128 129 sec := uint64(d / time.Second) 130 if todo.Bytes > 0 && sec > 0 && ticker { 131 bps = s.Bytes / sec 132 if s.Bytes >= todo.Bytes { 133 eta = 0 134 } else if bps > 0 { 135 eta = (todo.Bytes - s.Bytes) / bps 136 } 137 } 138 139 itemsDone := s.Files + s.Dirs 140 141 status1 := fmt.Sprintf("[%s] %s %s / %s %d / %d items %d errors ", 142 formatDuration(d), 143 formatPercent(s.Bytes, todo.Bytes), 144 formatBytes(s.Bytes), formatBytes(todo.Bytes), 145 itemsDone, itemsTodo, 146 s.Errors) 147 status2 := fmt.Sprintf("ETA %s ", formatSeconds(eta)) 148 149 if w := stdoutTerminalWidth(); w > 0 { 150 maxlen := w - len(status2) - 1 151 152 if maxlen < 4 { 153 status1 = "" 154 } else if len(status1) > maxlen { 155 status1 = status1[:maxlen-4] 156 status1 += "... " 157 } 158 } 159 160 PrintProgress("%s%s", status1, status2) 161 } 162 163 archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) { 164 fmt.Printf("\nduration: %s\n", formatDuration(d)) 165 } 166 167 return archiveProgress 168 } 169 170 func newArchiveStdinProgress(gopts GlobalOptions) *restic.Progress { 171 if gopts.Quiet { 172 return nil 173 } 174 175 archiveProgress := restic.NewProgress() 176 177 var bps uint64 178 179 archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) { 180 if IsProcessBackground() { 181 return 182 } 183 184 sec := uint64(d / time.Second) 185 if s.Bytes > 0 && sec > 0 && ticker { 186 bps = s.Bytes / sec 187 } 188 189 status1 := fmt.Sprintf("[%s] %s %s/s", formatDuration(d), 190 formatBytes(s.Bytes), 191 formatBytes(bps)) 192 193 if w := stdoutTerminalWidth(); w > 0 { 194 maxlen := w - len(status1) 195 196 if maxlen < 4 { 197 status1 = "" 198 } else if len(status1) > maxlen { 199 status1 = status1[:maxlen-4] 200 status1 += "... " 201 } 202 } 203 204 PrintProgress("%s", status1) 205 } 206 207 archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) { 208 fmt.Printf("\nduration: %s\n", formatDuration(d)) 209 } 210 211 return archiveProgress 212 } 213 214 // filterExisting returns a slice of all existing items, or an error if no 215 // items exist at all. 216 func filterExisting(items []string) (result []string, err error) { 217 for _, item := range items { 218 _, err := fs.Lstat(item) 219 if err != nil && os.IsNotExist(errors.Cause(err)) { 220 Warnf("%v does not exist, skipping\n", item) 221 continue 222 } 223 224 result = append(result, item) 225 } 226 227 if len(result) == 0 { 228 return nil, errors.Fatal("all target directories/files do not exist") 229 } 230 231 return 232 } 233 234 func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string) error { 235 if len(args) != 0 { 236 return errors.Fatal("when reading from stdin, no additional files can be specified") 237 } 238 239 fn := opts.StdinFilename 240 241 if fn == "" { 242 return errors.Fatal("filename for backup from stdin must not be empty") 243 } 244 245 if filepath.Base(fn) != fn || path.Base(fn) != fn { 246 return errors.Fatal("filename is invalid (may not contain a directory, slash or backslash)") 247 } 248 249 if gopts.password == "" { 250 return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD") 251 } 252 253 repo, err := OpenRepository(gopts) 254 if err != nil { 255 return err 256 } 257 258 lock, err := lockRepo(repo) 259 defer unlockRepo(lock) 260 if err != nil { 261 return err 262 } 263 264 err = repo.LoadIndex(gopts.ctx) 265 if err != nil { 266 return err 267 } 268 269 r := &archiver.Reader{ 270 Repository: repo, 271 Tags: opts.Tags, 272 Hostname: opts.Hostname, 273 } 274 275 _, id, err := r.Archive(gopts.ctx, fn, os.Stdin, newArchiveStdinProgress(gopts)) 276 if err != nil { 277 return err 278 } 279 280 Verbosef("archived as %v\n", id.Str()) 281 return nil 282 } 283 284 // readFromFile will read all lines from the given filename and write them to a 285 // string array, if filename is empty readFromFile returns and empty string 286 // array. If filename is a dash (-), readFromFile will read the lines from 287 // the standard input. 288 func readLinesFromFile(filename string) ([]string, error) { 289 if filename == "" { 290 return nil, nil 291 } 292 293 var r io.Reader = os.Stdin 294 if filename != "-" { 295 f, err := os.Open(filename) 296 if err != nil { 297 return nil, err 298 } 299 defer f.Close() 300 r = f 301 } 302 303 var lines []string 304 305 scanner := bufio.NewScanner(r) 306 for scanner.Scan() { 307 line := strings.TrimSpace(scanner.Text()) 308 // ignore empty lines 309 if line == "" { 310 continue 311 } 312 // strip comments 313 if strings.HasPrefix(line, "#") { 314 continue 315 } 316 lines = append(lines, line) 317 } 318 319 if err := scanner.Err(); err != nil { 320 return nil, err 321 } 322 323 return lines, nil 324 } 325 326 func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error { 327 if opts.FilesFrom == "-" && gopts.password == "" { 328 return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD") 329 } 330 331 fromfile, err := readLinesFromFile(opts.FilesFrom) 332 if err != nil { 333 return err 334 } 335 336 // merge files from files-from into normal args so we can reuse the normal 337 // args checks and have the ability to use both files-from and args at the 338 // same time 339 args = append(args, fromfile...) 340 if len(args) == 0 { 341 return errors.Fatal("nothing to backup, please specify target files/dirs") 342 } 343 344 target := make([]string, 0, len(args)) 345 for _, d := range args { 346 if a, err := filepath.Abs(d); err == nil { 347 d = a 348 } 349 target = append(target, d) 350 } 351 352 target, err = filterExisting(target) 353 if err != nil { 354 return err 355 } 356 357 // rejectFuncs collect functions that can reject items from the backup 358 var rejectFuncs []RejectFunc 359 360 // allowed devices 361 if opts.ExcludeOtherFS { 362 f, err := rejectByDevice(target) 363 if err != nil { 364 return err 365 } 366 rejectFuncs = append(rejectFuncs, f) 367 } 368 369 // add patterns from file 370 if len(opts.ExcludeFiles) > 0 { 371 opts.Excludes = append(opts.Excludes, readExcludePatternsFromFiles(opts.ExcludeFiles)...) 372 } 373 374 if len(opts.Excludes) > 0 { 375 rejectFuncs = append(rejectFuncs, rejectByPattern(opts.Excludes)) 376 } 377 378 if opts.ExcludeCaches { 379 opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55") 380 } 381 382 for _, spec := range opts.ExcludeIfPresent { 383 f, err := rejectIfPresent(spec) 384 if err != nil { 385 return err 386 } 387 388 rejectFuncs = append(rejectFuncs, f) 389 } 390 391 repo, err := OpenRepository(gopts) 392 if err != nil { 393 return err 394 } 395 396 lock, err := lockRepo(repo) 397 defer unlockRepo(lock) 398 if err != nil { 399 return err 400 } 401 402 // exclude restic cache 403 if repo.Cache != nil { 404 f, err := rejectResticCache(repo) 405 if err != nil { 406 return err 407 } 408 409 rejectFuncs = append(rejectFuncs, f) 410 } 411 412 err = repo.LoadIndex(gopts.ctx) 413 if err != nil { 414 return err 415 } 416 417 var parentSnapshotID *restic.ID 418 419 // Force using a parent 420 if !opts.Force && opts.Parent != "" { 421 id, err := restic.FindSnapshot(repo, opts.Parent) 422 if err != nil { 423 return errors.Fatalf("invalid id %q: %v", opts.Parent, err) 424 } 425 426 parentSnapshotID = &id 427 } 428 429 // Find last snapshot to set it as parent, if not already set 430 if !opts.Force && parentSnapshotID == nil { 431 id, err := restic.FindLatestSnapshot(gopts.ctx, repo, target, []restic.TagList{}, opts.Hostname) 432 if err == nil { 433 parentSnapshotID = &id 434 } else if err != restic.ErrNoSnapshotFound { 435 return err 436 } 437 } 438 439 if parentSnapshotID != nil { 440 Verbosef("using parent snapshot %v\n", parentSnapshotID.Str()) 441 } 442 443 Verbosef("scan %v\n", target) 444 445 selectFilter := func(item string, fi os.FileInfo) bool { 446 for _, reject := range rejectFuncs { 447 if reject(item, fi) { 448 return false 449 } 450 } 451 return true 452 } 453 454 stat, err := archiver.Scan(target, selectFilter, newScanProgress(gopts)) 455 if err != nil { 456 return err 457 } 458 459 arch := archiver.New(repo) 460 arch.Excludes = opts.Excludes 461 arch.SelectFilter = selectFilter 462 arch.WithAccessTime = opts.WithAtime 463 464 arch.Warn = func(dir string, fi os.FileInfo, err error) { 465 // TODO: make ignoring errors configurable 466 Warnf("%s\rwarning for %s: %v\n", ClearLine(), dir, err) 467 } 468 469 timeStamp := time.Now() 470 if opts.TimeStamp != "" { 471 timeStamp, err = time.Parse(TimeFormat, opts.TimeStamp) 472 if err != nil { 473 return errors.Fatalf("error in time option: %v\n", err) 474 } 475 } 476 477 _, id, err := arch.Snapshot(gopts.ctx, newArchiveProgress(gopts, stat), target, opts.Tags, opts.Hostname, parentSnapshotID, timeStamp) 478 if err != nil { 479 return err 480 } 481 482 Verbosef("snapshot %s saved\n", id.Str()) 483 484 return nil 485 } 486 487 func readExcludePatternsFromFiles(excludeFiles []string) []string { 488 var excludes []string 489 for _, filename := range excludeFiles { 490 err := func() (err error) { 491 file, err := fs.Open(filename) 492 if err != nil { 493 return err 494 } 495 defer func() { 496 // return pre-close error if there was one 497 if errClose := file.Close(); err == nil { 498 err = errClose 499 } 500 }() 501 502 scanner := bufio.NewScanner(file) 503 for scanner.Scan() { 504 line := strings.TrimSpace(scanner.Text()) 505 506 // ignore empty lines 507 if line == "" { 508 continue 509 } 510 511 // strip comments 512 if strings.HasPrefix(line, "#") { 513 continue 514 } 515 516 line = os.ExpandEnv(line) 517 excludes = append(excludes, line) 518 } 519 return scanner.Err() 520 }() 521 if err != nil { 522 Warnf("error reading exclude patterns: %v:", err) 523 return nil 524 } 525 } 526 return excludes 527 }