github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/cmd/bisync/resync.go (about) 1 package bisync 2 3 import ( 4 "context" 5 "os" 6 "path/filepath" 7 8 "github.com/rclone/rclone/cmd/bisync/bilib" 9 "github.com/rclone/rclone/fs" 10 "github.com/rclone/rclone/fs/filter" 11 "github.com/rclone/rclone/lib/terminal" 12 ) 13 14 // for backward compatibility, --resync is now equivalent to --resync-mode path1 15 // and either flag is sufficient without the other. 16 func (b *bisyncRun) setResyncDefaults() { 17 if b.opt.Resync && b.opt.ResyncMode == PreferNone { 18 fs.Debugf(nil, Color(terminal.Dim, "defaulting to --resync-mode path1 as --resync is set")) 19 b.opt.ResyncMode = PreferPath1 20 } 21 if b.opt.ResyncMode != PreferNone { 22 b.opt.Resync = true 23 Opt.Resync = true // shouldn't be using this one, but set to be safe 24 } 25 26 // checks and warnings 27 if (b.opt.ResyncMode == PreferNewer || b.opt.ResyncMode == PreferOlder) && (b.fs1.Precision() == fs.ModTimeNotSupported || b.fs2.Precision() == fs.ModTimeNotSupported) { 28 fs.Logf(nil, Color(terminal.YellowFg, "WARNING: ignoring --resync-mode %s as at least one remote does not support modtimes."), b.opt.ResyncMode.String()) 29 b.opt.ResyncMode = PreferPath1 30 } else if (b.opt.ResyncMode == PreferNewer || b.opt.ResyncMode == PreferOlder) && !b.opt.Compare.Modtime { 31 fs.Logf(nil, Color(terminal.YellowFg, "WARNING: ignoring --resync-mode %s as --compare does not include modtime."), b.opt.ResyncMode.String()) 32 b.opt.ResyncMode = PreferPath1 33 } 34 if (b.opt.ResyncMode == PreferLarger || b.opt.ResyncMode == PreferSmaller) && !b.opt.Compare.Size { 35 fs.Logf(nil, Color(terminal.YellowFg, "WARNING: ignoring --resync-mode %s as --compare does not include size."), b.opt.ResyncMode.String()) 36 b.opt.ResyncMode = PreferPath1 37 } 38 } 39 40 // resync implements the --resync mode. 41 // It will generate path1 and path2 listings, 42 // copy any unique files to the opposite path, 43 // and resolve any differing files according to the --resync-mode. 44 func (b *bisyncRun) resync(octx, fctx context.Context) error { 45 fs.Infof(nil, "Copying Path2 files to Path1") 46 47 // Save blank filelists (will be filled from sync results) 48 var ls1 = newFileList() 49 var ls2 = newFileList() 50 err = ls1.save(fctx, b.newListing1) 51 if err != nil { 52 b.handleErr(ls1, "error saving ls1 from resync", err, true, true) 53 b.abort = true 54 } 55 err = ls2.save(fctx, b.newListing2) 56 if err != nil { 57 b.handleErr(ls2, "error saving ls2 from resync", err, true, true) 58 b.abort = true 59 } 60 61 // Check access health on the Path1 and Path2 filesystems 62 // enforce even though this is --resync 63 if b.opt.CheckAccess { 64 fs.Infof(nil, "Checking access health") 65 66 filesNow1, filesNow2, err := b.findCheckFiles(fctx) 67 if err != nil { 68 b.critical = true 69 b.retryable = true 70 return err 71 } 72 73 ds1 := &deltaSet{ 74 checkFiles: bilib.Names{}, 75 } 76 77 ds2 := &deltaSet{ 78 checkFiles: bilib.Names{}, 79 } 80 81 for _, file := range filesNow1.list { 82 if filepath.Base(file) == b.opt.CheckFilename { 83 ds1.checkFiles.Add(file) 84 } 85 } 86 87 for _, file := range filesNow2.list { 88 if filepath.Base(file) == b.opt.CheckFilename { 89 ds2.checkFiles.Add(file) 90 } 91 } 92 93 err = b.checkAccess(ds1.checkFiles, ds2.checkFiles) 94 if err != nil { 95 b.critical = true 96 b.retryable = true 97 return err 98 } 99 } 100 101 var results2to1 []Results 102 var results1to2 []Results 103 queues := queues{} 104 105 b.indent("Path2", "Path1", "Resync is copying files to") 106 ctxRun := b.opt.setDryRun(fctx) 107 // fctx has our extra filters added! 108 ctxSync, filterSync := filter.AddConfig(ctxRun) 109 if filterSync.Opt.MinSize == -1 { 110 fs.Debugf(nil, "filterSync.Opt.MinSize: %v", filterSync.Opt.MinSize) 111 } 112 b.resyncIs1to2 = false 113 ctxSync = b.setResyncConfig(ctxSync) 114 ctxSync = b.setBackupDir(ctxSync, 1) 115 // 2 to 1 116 if results2to1, err = b.resyncDir(ctxSync, b.fs2, b.fs1); err != nil { 117 b.critical = true 118 return err 119 } 120 121 b.indent("Path1", "Path2", "Resync is copying files to") 122 b.resyncIs1to2 = true 123 ctxSync = b.setResyncConfig(ctxSync) 124 ctxSync = b.setBackupDir(ctxSync, 2) 125 // 1 to 2 126 if results1to2, err = b.resyncDir(ctxSync, b.fs1, b.fs2); err != nil { 127 b.critical = true 128 return err 129 } 130 131 fs.Infof(nil, "Resync updating listings") 132 b.saveOldListings() // may not exist, as this is --resync 133 b.replaceCurrentListings() 134 135 resultsToQueue := func(results []Results) bilib.Names { 136 names := bilib.Names{} 137 for _, result := range results { 138 if result.Name != "" && 139 (result.Flags != "d" || b.opt.CreateEmptySrcDirs) && 140 result.IsSrc && result.Src != "" && 141 (result.Winner.Err == nil || result.Flags == "d") { 142 names.Add(result.Name) 143 } 144 } 145 return names 146 } 147 148 // resync 2to1 149 queues.copy2to1 = resultsToQueue(results2to1) 150 if err = b.modifyListing(fctx, b.fs2, b.fs1, results2to1, queues, false); err != nil { 151 b.critical = true 152 return err 153 } 154 155 // resync 1to2 156 queues.copy1to2 = resultsToQueue(results1to2) 157 if err = b.modifyListing(fctx, b.fs1, b.fs2, results1to2, queues, true); err != nil { 158 b.critical = true 159 return err 160 } 161 162 if b.opt.CheckSync == CheckSyncTrue && !b.opt.DryRun { 163 path1 := bilib.FsPath(b.fs1) 164 path2 := bilib.FsPath(b.fs2) 165 fs.Infof(nil, "Validating listings for Path1 %s vs Path2 %s", quotePath(path1), quotePath(path2)) 166 if err := b.checkSync(b.listing1, b.listing2); err != nil { 167 b.critical = true 168 return err 169 } 170 } 171 172 if !b.opt.NoCleanup { 173 _ = os.Remove(b.newListing1) 174 _ = os.Remove(b.newListing2) 175 } 176 return nil 177 } 178 179 /* 180 --resync-mode implementation: 181 PreferPath1: set ci.IgnoreExisting true, then false 182 PreferPath2: set ci.IgnoreExisting false, then true 183 PreferNewer: set ci.UpdateOlder in both directions 184 PreferOlder: override EqualFn to implement custom logic 185 PreferLarger: override EqualFn to implement custom logic 186 PreferSmaller: override EqualFn to implement custom logic 187 */ 188 func (b *bisyncRun) setResyncConfig(ctx context.Context) context.Context { 189 ci := fs.GetConfig(ctx) 190 switch b.opt.ResyncMode { 191 case PreferPath1: 192 if !b.resyncIs1to2 { // 2to1 (remember 2to1 is first) 193 ci.IgnoreExisting = true 194 } else { // 1to2 195 ci.IgnoreExisting = false 196 } 197 case PreferPath2: 198 if !b.resyncIs1to2 { // 2to1 (remember 2to1 is first) 199 ci.IgnoreExisting = false 200 } else { // 1to2 201 ci.IgnoreExisting = true 202 } 203 case PreferNewer: 204 ci.UpdateOlder = true 205 } 206 // for older, larger, and smaller, we return it unchanged and handle it later 207 return ctx 208 } 209 210 func (b *bisyncRun) resyncWhichIsWhich(src, dst fs.ObjectInfo) (path1, path2 fs.ObjectInfo) { 211 if b.resyncIs1to2 { 212 return src, dst 213 } 214 return dst, src 215 } 216 217 // equal in this context really means "don't transfer", so we should 218 // return true if the files are actually equal or if dest is winner, 219 // false if src is winner 220 // When can't determine, we end up running the normal Equal() to tie-break (due to our differ functions). 221 func (b *bisyncRun) resyncWinningPathToEqual(winningPath int) bool { 222 if b.resyncIs1to2 { 223 return winningPath != 1 224 } 225 return winningPath != 2 226 }