github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/cmd/serve/restic/restic.go (about) 1 // Package restic serves a remote suitable for use with restic 2 package restic 3 4 import ( 5 "encoding/json" 6 "errors" 7 "net/http" 8 "os" 9 "path" 10 "regexp" 11 "strings" 12 "time" 13 14 "github.com/rclone/rclone/cmd" 15 "github.com/rclone/rclone/cmd/serve/httplib" 16 "github.com/rclone/rclone/cmd/serve/httplib/httpflags" 17 "github.com/rclone/rclone/cmd/serve/httplib/serve" 18 "github.com/rclone/rclone/fs" 19 "github.com/rclone/rclone/fs/accounting" 20 "github.com/rclone/rclone/fs/config/flags" 21 "github.com/rclone/rclone/fs/fserrors" 22 "github.com/rclone/rclone/fs/operations" 23 "github.com/rclone/rclone/fs/walk" 24 "github.com/spf13/cobra" 25 "golang.org/x/crypto/ssh/terminal" 26 "golang.org/x/net/http2" 27 ) 28 29 var ( 30 stdio bool 31 appendOnly bool 32 privateRepos bool 33 ) 34 35 func init() { 36 httpflags.AddFlags(Command.Flags()) 37 flagSet := Command.Flags() 38 flags.BoolVarP(flagSet, &stdio, "stdio", "", false, "run an HTTP2 server on stdin/stdout") 39 flags.BoolVarP(flagSet, &appendOnly, "append-only", "", false, "disallow deletion of repository data") 40 flags.BoolVarP(flagSet, &privateRepos, "private-repos", "", false, "users can only access their private repo") 41 } 42 43 // Command definition for cobra 44 var Command = &cobra.Command{ 45 Use: "restic remote:path", 46 Short: `Serve the remote for restic's REST API.`, 47 Long: `rclone serve restic implements restic's REST backend API 48 over HTTP. This allows restic to use rclone as a data storage 49 mechanism for cloud providers that restic does not support directly. 50 51 [Restic](https://restic.net/) is a command line program for doing 52 backups. 53 54 The server will log errors. Use -v to see access logs. 55 56 --bwlimit will be respected for file transfers. Use --stats to 57 control the stats printing. 58 59 ### Setting up rclone for use by restic ### 60 61 First [set up a remote for your chosen cloud provider](/docs/#configure). 62 63 Once you have set up the remote, check it is working with, for example 64 "rclone lsd remote:". You may have called the remote something other 65 than "remote:" - just substitute whatever you called it in the 66 following instructions. 67 68 Now start the rclone restic server 69 70 rclone serve restic -v remote:backup 71 72 Where you can replace "backup" in the above by whatever path in the 73 remote you wish to use. 74 75 By default this will serve on "localhost:8080" you can change this 76 with use of the "--addr" flag. 77 78 You might wish to start this server on boot. 79 80 ### Setting up restic to use rclone ### 81 82 Now you can [follow the restic 83 instructions](http://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) 84 on setting up restic. 85 86 Note that you will need restic 0.8.2 or later to interoperate with 87 rclone. 88 89 For the example above you will want to use "http://localhost:8080/" as 90 the URL for the REST server. 91 92 For example: 93 94 $ export RESTIC_REPOSITORY=rest:http://localhost:8080/ 95 $ export RESTIC_PASSWORD=yourpassword 96 $ restic init 97 created restic backend 8b1a4b56ae at rest:http://localhost:8080/ 98 99 Please note that knowledge of your password is required to access 100 the repository. Losing your password means that your data is 101 irrecoverably lost. 102 $ restic backup /path/to/files/to/backup 103 scan [/path/to/files/to/backup] 104 scanned 189 directories, 312 files in 0:00 105 [0:00] 100.00% 38.128 MiB / 38.128 MiB 501 / 501 items 0 errors ETA 0:00 106 duration: 0:00 107 snapshot 45c8fdd8 saved 108 109 #### Multiple repositories #### 110 111 Note that you can use the endpoint to host multiple repositories. Do 112 this by adding a directory name or path after the URL. Note that 113 these **must** end with /. Eg 114 115 $ export RESTIC_REPOSITORY=rest:http://localhost:8080/user1repo/ 116 # backup user1 stuff 117 $ export RESTIC_REPOSITORY=rest:http://localhost:8080/user2repo/ 118 # backup user2 stuff 119 120 #### Private repositories #### 121 122 The "--private-repos" flag can be used to limit users to repositories starting 123 with a path of ` + "`/<username>/`" + `. 124 ` + httplib.Help, 125 Run: func(command *cobra.Command, args []string) { 126 cmd.CheckArgs(1, 1, command, args) 127 f := cmd.NewFsSrc(args) 128 cmd.Run(false, true, command, func() error { 129 s := newServer(f, &httpflags.Opt) 130 if stdio { 131 if terminal.IsTerminal(int(os.Stdout.Fd())) { 132 return errors.New("Refusing to run HTTP2 server directly on a terminal, please let restic start rclone") 133 } 134 135 conn := &StdioConn{ 136 stdin: os.Stdin, 137 stdout: os.Stdout, 138 } 139 140 httpSrv := &http2.Server{} 141 opts := &http2.ServeConnOpts{ 142 Handler: http.HandlerFunc(s.handler), 143 } 144 httpSrv.ServeConn(conn, opts) 145 return nil 146 } 147 err := s.Serve() 148 if err != nil { 149 return err 150 } 151 s.Wait() 152 return nil 153 }) 154 }, 155 } 156 157 const ( 158 resticAPIV2 = "application/vnd.x.restic.rest.v2" 159 ) 160 161 // server contains everything to run the server 162 type server struct { 163 *httplib.Server 164 f fs.Fs 165 } 166 167 func newServer(f fs.Fs, opt *httplib.Options) *server { 168 mux := http.NewServeMux() 169 s := &server{ 170 Server: httplib.NewServer(mux, opt), 171 f: f, 172 } 173 mux.HandleFunc(s.Opt.BaseURL+"/", s.handler) 174 return s 175 } 176 177 // Serve runs the http server in the background. 178 // 179 // Use s.Close() and s.Wait() to shutdown server 180 func (s *server) Serve() error { 181 err := s.Server.Serve() 182 if err != nil { 183 return err 184 } 185 fs.Logf(s.f, "Serving restic REST API on %s", s.URL()) 186 return nil 187 } 188 189 var matchData = regexp.MustCompile("(?:^|/)data/([^/]{2,})$") 190 191 // Makes a remote from a URL path. This implements the backend layout 192 // required by restic. 193 func makeRemote(path string) string { 194 path = strings.Trim(path, "/") 195 parts := matchData.FindStringSubmatch(path) 196 // if no data directory, layout is flat 197 if parts == nil { 198 return path 199 } 200 // otherwise map 201 // data/2159dd48 to 202 // data/21/2159dd48 203 fileName := parts[1] 204 prefix := path[:len(path)-len(fileName)] 205 return prefix + fileName[:2] + "/" + fileName 206 } 207 208 // handler reads incoming requests and dispatches them 209 func (s *server) handler(w http.ResponseWriter, r *http.Request) { 210 w.Header().Set("Accept-Ranges", "bytes") 211 w.Header().Set("Server", "rclone/"+fs.Version) 212 213 path, ok := s.Path(w, r) 214 if !ok { 215 return 216 } 217 remote := makeRemote(path) 218 fs.Debugf(s.f, "%s %s", r.Method, path) 219 220 v := r.Context().Value(httplib.ContextUserKey) 221 if privateRepos && (v == nil || !strings.HasPrefix(path, "/"+v.(string)+"/")) { 222 http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) 223 return 224 } 225 226 // Dispatch on path then method 227 if strings.HasSuffix(path, "/") { 228 switch r.Method { 229 case "GET": 230 s.listObjects(w, r, remote) 231 case "POST": 232 s.createRepo(w, r, remote) 233 default: 234 http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) 235 } 236 } else { 237 switch r.Method { 238 case "GET", "HEAD": 239 s.serveObject(w, r, remote) 240 case "POST": 241 s.postObject(w, r, remote) 242 case "DELETE": 243 s.deleteObject(w, r, remote) 244 default: 245 http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) 246 } 247 } 248 } 249 250 // get the remote 251 func (s *server) serveObject(w http.ResponseWriter, r *http.Request, remote string) { 252 o, err := s.f.NewObject(r.Context(), remote) 253 if err != nil { 254 fs.Debugf(remote, "%s request error: %v", r.Method, err) 255 http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 256 return 257 } 258 serve.Object(w, r, o) 259 } 260 261 // postObject posts an object to the repository 262 func (s *server) postObject(w http.ResponseWriter, r *http.Request, remote string) { 263 if appendOnly { 264 // make sure the file does not exist yet 265 _, err := s.f.NewObject(r.Context(), remote) 266 if err == nil { 267 fs.Errorf(remote, "Post request: file already exists, refusing to overwrite in append-only mode") 268 http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) 269 270 return 271 } 272 } 273 274 _, err := operations.RcatSize(r.Context(), s.f, remote, r.Body, r.ContentLength, time.Now()) 275 if err != nil { 276 err = accounting.Stats(r.Context()).Error(err) 277 fs.Errorf(remote, "Post request rcat error: %v", err) 278 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 279 280 return 281 } 282 } 283 284 // delete the remote 285 func (s *server) deleteObject(w http.ResponseWriter, r *http.Request, remote string) { 286 if appendOnly { 287 parts := strings.Split(r.URL.Path, "/") 288 289 // if path doesn't end in "/locks/:name", disallow the operation 290 if len(parts) < 2 || parts[len(parts)-2] != "locks" { 291 http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) 292 return 293 } 294 } 295 296 o, err := s.f.NewObject(r.Context(), remote) 297 if err != nil { 298 fs.Debugf(remote, "Delete request error: %v", err) 299 http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 300 return 301 } 302 303 if err := o.Remove(r.Context()); err != nil { 304 fs.Errorf(remote, "Delete request remove error: %v", err) 305 if err == fs.ErrorObjectNotFound { 306 http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 307 } else { 308 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 309 } 310 return 311 } 312 } 313 314 // listItem is an element returned for the restic v2 list response 315 type listItem struct { 316 Name string `json:"name"` 317 Size int64 `json:"size"` 318 } 319 320 // return type for list 321 type listItems []listItem 322 323 // add a DirEntry to the listItems 324 func (ls *listItems) add(entry fs.DirEntry) { 325 if o, ok := entry.(fs.Object); ok { 326 *ls = append(*ls, listItem{ 327 Name: path.Base(o.Remote()), 328 Size: o.Size(), 329 }) 330 } 331 } 332 333 // listObjects lists all Objects of a given type in an arbitrary order. 334 func (s *server) listObjects(w http.ResponseWriter, r *http.Request, remote string) { 335 fs.Debugf(remote, "list request") 336 337 if r.Header.Get("Accept") != resticAPIV2 { 338 fs.Errorf(remote, "Restic v2 API required") 339 http.Error(w, "Restic v2 API required", http.StatusBadRequest) 340 return 341 } 342 343 // make sure an empty list is returned, and not a 'nil' value 344 ls := listItems{} 345 346 // if remote supports ListR use that directly, otherwise use recursive Walk 347 err := walk.ListR(r.Context(), s.f, remote, true, -1, walk.ListObjects, func(entries fs.DirEntries) error { 348 for _, entry := range entries { 349 ls.add(entry) 350 } 351 return nil 352 }) 353 if err != nil { 354 _, err = fserrors.Cause(err) 355 if err != fs.ErrorDirNotFound { 356 fs.Errorf(remote, "list failed: %#v %T", err, err) 357 http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 358 return 359 } 360 } 361 362 w.Header().Set("Content-Type", "application/vnd.x.restic.rest.v2") 363 enc := json.NewEncoder(w) 364 err = enc.Encode(ls) 365 if err != nil { 366 fs.Errorf(remote, "failed to write list: %v", err) 367 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 368 return 369 } 370 } 371 372 // createRepo creates repository directories. 373 // 374 // We don't bother creating the data dirs as rclone will create them on the fly 375 func (s *server) createRepo(w http.ResponseWriter, r *http.Request, remote string) { 376 fs.Infof(remote, "Creating repository") 377 378 if r.URL.Query().Get("create") != "true" { 379 http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 380 return 381 } 382 383 err := s.f.Mkdir(r.Context(), remote) 384 if err != nil { 385 fs.Errorf(remote, "Create repo failed to Mkdir: %v", err) 386 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 387 return 388 } 389 390 for _, name := range []string{"data", "index", "keys", "locks", "snapshots"} { 391 dirRemote := path.Join(remote, name) 392 err := s.f.Mkdir(r.Context(), dirRemote) 393 if err != nil { 394 fs.Errorf(dirRemote, "Create repo failed to Mkdir: %v", err) 395 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 396 return 397 } 398 } 399 }