exp.upspin.io@v0.0.0-20230625230448-5076e5b595ec/cmd/upsync/upsync.go (about) 1 // Copyright 2019 The Upspin Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Upsync keeps a local disk copy in sync with a master version in Upspin. 6 // See the command's usage method for documentation. 7 package main 8 9 import ( 10 "flag" 11 "fmt" 12 "io/ioutil" 13 "log" 14 "os" 15 "path/filepath" 16 "strings" 17 "time" 18 19 "upspin.io/client" 20 "upspin.io/cmd/cacheserver/cacheutil" 21 "upspin.io/config" 22 "upspin.io/flags" 23 "upspin.io/transports" 24 "upspin.io/upspin" 25 "upspin.io/version" 26 ) 27 28 var lastUpsync int64 // Unix time when an upsync was last completed 29 30 const help = `Upsync keeps a local disk copy in sync with a master version in 31 Upspin. It is a weak substitute for upspinfs. 32 33 To start, create a local directory whose path ends in a string that looks like 34 an existing upspin directory, such as ~/u/alice@example.com. Cd there and execute 35 upsync. Make local edits to the downloaded files or create new files, and then 36 upsync to upload your changes to the Upspin master. To discard your local changes, 37 just remove the edited local files and upsync. (Executing both local rm and 38 upspin rm are required to remove content permanently.) 39 40 Upsync prints which files it is uploading or downloading and declines to download 41 files larger than 50MB. It promises never to write outside the starting directory 42 and subdirectories and, as an initial way to enforce that, declines all symlinks. 43 44 There are no clever merge heuristics; copying back and forth proceeds by a trivial 45 "newest wins" rule. This requires some discipline in remembering to upsync after 46 each editing session and is better suited to single person rather than joint 47 editing. Don't let your computer clocks drift. 48 49 With better FUSE support on Windows and OpenBSD it will be possible to switch 50 to the much preferable upspinfs. But even then upsync may have benefits: 51 * enables work offline, i.e. a workaround for (missing) distributed upspinfs 52 * offers mitigation of user misfortune, such as losing upspin keys 53 * provides a worked out example for new Upspin client developers 54 * leaves a backup in case cloud store or Upspin projects die without warning 55 56 This tool was written assuming you are an experienced Upspin user trying to 57 assist a friend with file sharing or backup on Windows 10. Here is a checklist: 58 1. create or check existing upspin account and permissions 59 It is helpful if you can provide them space on an existing server. 60 2. confirm \Users\alice\upspin\config is correct 61 3. disk must be NTFS (because FAT has peculiar timestamps) 62 4. open a powershell window 63 5. install go and git, if not already there 64 6. go get -u upspin.io/cmd/... 65 7. fetch upsync.go; go install 66 Go files must be transferred as UTF8, else expect a NUL compile warning. 67 8. mkdir \Users\alice\u\alice@example.com 68 9. upsync 69 70 ` 71 72 const cmdName = "upsync" 73 74 var upsyncFlag = flag.String("upsync", upspinDir("upsync"), "file whose mtime is last upsync") 75 76 func usage() { 77 fmt.Fprintln(os.Stderr, help) 78 fmt.Fprintf(os.Stderr, "Usage: %s [flags]\n", os.Args[0]) 79 flag.PrintDefaults() 80 } 81 82 func main() { 83 log.SetFlags(0) 84 log.SetPrefix("upsync: ") 85 flag.Usage = usage 86 flags.Parse(flags.Client, "version") 87 if flags.Version { 88 fmt.Print(version.Version()) 89 return 90 } 91 if flag.NArg() > 0 { 92 usage() 93 os.Exit(2) 94 } 95 96 err := do() 97 if err != nil { 98 log.Fatal(err) 99 } 100 } 101 102 func do() error { 103 // Setup Upspin client. 104 cfg, err := config.FromFile(flags.Config) 105 if err != nil { 106 return err 107 } 108 transports.Init(cfg) 109 cacheutil.Start(cfg) 110 upc := client.New(cfg) 111 112 // Guess at previous upsync time. 113 getwd, err := os.Getwd() 114 if err != nil { 115 return err 116 } 117 lastUpsyncFi, err := os.Stat(*upsyncFlag) 118 if os.IsNotExist(err) { // first time 119 err = ioutil.WriteFile(*upsyncFlag, []byte(getwd), 0644) 120 if err != nil { 121 return err 122 } 123 } else if err != nil { // stat failed; very unusual 124 return err 125 } else { // normal case 126 lastUpsync = lastUpsyncFi.ModTime().Unix() 127 } 128 if lastUpsyncFi != nil { 129 log.Printf("lastUpsync %v", lastUpsyncFi.ModTime()) 130 } 131 132 // Find first component of current directory that looks like email address, 133 // then make wd == upspin working directory. 134 wd := getwd 135 i := strings.IndexByte(wd, '@') 136 if i < 0 { 137 return fmt.Errorf("couldn't find upspin user name in working directory %s", getwd) 138 } 139 i = strings.LastIndexAny(wd[:i], "\\/") 140 if i < 0 { 141 return fmt.Errorf("unable to parse working directory %s", getwd) 142 } 143 slash := wd[i : i+1] 144 wd = wd[i+1:] 145 if slash != "/" { 146 wd = strings.ReplaceAll(wd, slash, "/") 147 } 148 149 // Start copying. 150 err = upsync(upc, wd, "") 151 if err != nil { 152 return err 153 } 154 155 // Save time of this upsync for next upsync "skipping old" heuristic. 156 err = ioutil.WriteFile(*upsyncFlag, []byte(getwd), 0644) 157 // We're more or less successful even if we can't record the time. But warn. 158 return err 159 } 160 161 // upsync walks the local and remote trees rooted at subdir to update each file to newer versions. 162 // The upspin.Client upc and the Upspin starting directory wd don't change from what was set in main. 163 // The subdir argument changes for the depth-first recursive tree walk and is either empty or a 164 // directory pathname with trailing slash. 165 func upsync(upc upspin.Client, wd, subdir string) error { 166 167 // udir and ldir are sorted lists of remote and local files in subdir. 168 udir, err := upc.Glob(wd + "/" + subdir + "*") 169 if err != nil { 170 return err 171 } 172 ldir, err := ioutil.ReadDir(subdir + ".") 173 if err != nil { 174 return err 175 } 176 177 // Advance through the two lists, comparing at each iteration udir[uj] and ldir[lj]. 178 uj := 0 179 lj := 0 180 for { 181 cmp := 0 // -1,0,1 as udir[uj] sorts before,same,after ldir[lj] 182 if lj < len(ldir) && ldir[lj].Mode()&os.ModeSymlink != 0 { 183 return fmt.Errorf("local symlinks are not allowed: %s", ldir[lj].Name()) 184 } 185 if uj >= len(udir) { 186 if lj >= len(ldir) { 187 break // both lists exhausted 188 } 189 cmp = 1 190 } else if lj >= len(ldir) { 191 cmp = -1 192 } else { 193 cmp = strings.Compare(string(udir[uj].SignedName)[len(wd)+1:], subdir+ldir[lj].Name()) 194 } 195 196 // Copy newer to older/missing. 197 switch cmp { 198 case -1: 199 pathname := string(udir[uj].SignedName)[len(wd)+1:] 200 switch { 201 case udir[uj].Attr&upspin.AttrLink != 0: 202 fmt.Println("ignoring upspin symlink", pathname) 203 case udir[uj].Attr&upspin.AttrDirectory != 0: 204 err = os.Mkdir(pathname, 0700) 205 if err != nil { 206 return err 207 } 208 err = upsync(upc, wd, pathname+"/") 209 if err != nil { 210 return err 211 } 212 case udir[uj].Attr&upspin.AttrIncomplete != 0: 213 fmt.Println("permission problem; creating placeholder ", pathname) 214 empty := make([]byte, 0) 215 err = ioutil.WriteFile(pathname, empty, 0) 216 if err != nil { 217 return err 218 } 219 case len(udir[uj].Blocks) > 50: 220 fmt.Println("skipping big", pathname) 221 default: 222 utime := int64(udir[uj].Time) 223 err = pull(upc, wd, pathname, utime) 224 if err != nil { 225 return err 226 } 227 } 228 uj++ 229 case 0: 230 pathname := subdir + ldir[lj].Name() 231 uIsDir := udir[uj].Attr&upspin.AttrDirectory != 0 232 lIsDir := ldir[lj].IsDir() 233 if uIsDir != lIsDir { 234 return fmt.Errorf("same name, different Directory attribute! %s", pathname) 235 } 236 if uIsDir { 237 err = upsync(upc, wd, pathname+"/") 238 if err != nil { 239 return err 240 } 241 } else { 242 utime := int64(udir[uj].Time) 243 ltime := ldir[lj].ModTime().Unix() 244 if utime > ltime { 245 err = pull(upc, wd, pathname, utime) 246 if err != nil { 247 return err 248 } 249 } else if utime < ltime { 250 err = push(upc, wd, pathname, ltime) 251 if err != nil { 252 return err 253 } 254 } else { 255 // Assume already in sync. 256 // TODO(ehg) Compare sizes as sanity check? 257 } 258 } 259 uj++ 260 lj++ 261 case 1: 262 pathname := subdir + ldir[lj].Name() 263 if ldir[lj].IsDir() { 264 fmt.Println("upspin mkdir", wd+"/"+pathname) 265 _, err = upc.MakeDirectory(upspin.PathName(wd + "/" + pathname)) 266 if err != nil { 267 return err 268 } 269 err = upsync(upc, wd, pathname+"/") 270 if err != nil { 271 return err 272 } 273 } else { 274 ltime := ldir[lj].ModTime().Unix() 275 err = push(upc, wd, pathname, ltime) 276 if err != nil { 277 return err 278 } 279 } 280 lj++ 281 } 282 } 283 return nil 284 } 285 286 // pull copies pathname from Upspin to local disk, copying the modification time. 287 func pull(upc upspin.Client, wd, pathname string, utime int64) error { 288 fmt.Println("pull", pathname) 289 // TODO(ehg) If we ever decide to parallelize, or even if we decide to 290 // run on small memory machines, switch to io.Copy(). 291 bytes, err := upc.Get(upspin.PathName(wd + "/" + pathname)) 292 if err != nil { 293 return err 294 } 295 err = ioutil.WriteFile(pathname, bytes, 0600) 296 if err != nil { 297 return err 298 } 299 mtime := time.Unix(utime, 0) 300 err = os.Chtimes(pathname, mtime, mtime) 301 if err != nil { 302 return err 303 } 304 return nil 305 } 306 307 // pull copies pathname from local disk to Upspin, copying the modification time. 308 func push(upc upspin.Client, wd, pathname string, ltime int64) error { 309 if ltime < lastUpsync { 310 fmt.Printf("skipping old %v %v\n", pathname, ltime) 311 return nil 312 } 313 fmt.Println("push", pathname) 314 bytes, err := ioutil.ReadFile(pathname) 315 if err != nil { 316 return err 317 } 318 path := upspin.PathName(wd + "/" + pathname) 319 _, err = upc.Put(path, bytes) 320 if err != nil { 321 return err 322 } 323 err = upc.SetTime(path, upspin.Time(ltime)) 324 if err != nil { 325 return err 326 } 327 return nil 328 } 329 330 // upspinDir is copied from upspin.io/flags/flags.go. 331 func upspinDir(subdir string) string { 332 home, err := config.Homedir() 333 if err != nil { 334 log.Printf("upsync: could not locate home directory: %v", err) 335 home = "." 336 } 337 return filepath.Join(home, "upspin", subdir) 338 }