github.com/artpar/rclone@v1.67.3/cmd/serve/restic/restic.go (about) 1 // Package restic serves a remote suitable for use with restic 2 package restic 3 4 import ( 5 "context" 6 "encoding/json" 7 "errors" 8 "fmt" 9 "net/http" 10 "os" 11 "path" 12 "regexp" 13 "strings" 14 "time" 15 16 "github.com/artpar/rclone/cmd" 17 "github.com/artpar/rclone/fs" 18 "github.com/artpar/rclone/fs/accounting" 19 "github.com/artpar/rclone/fs/config/flags" 20 "github.com/artpar/rclone/fs/operations" 21 "github.com/artpar/rclone/fs/walk" 22 libhttp "github.com/artpar/rclone/lib/http" 23 "github.com/artpar/rclone/lib/http/serve" 24 "github.com/artpar/rclone/lib/systemd" 25 "github.com/artpar/rclone/lib/terminal" 26 "github.com/go-chi/chi/v5" 27 "github.com/go-chi/chi/v5/middleware" 28 "github.com/spf13/cobra" 29 "golang.org/x/net/http2" 30 ) 31 32 // Options required for http server 33 type Options struct { 34 Auth libhttp.AuthConfig 35 HTTP libhttp.Config 36 Stdio bool 37 AppendOnly bool 38 PrivateRepos bool 39 CacheObjects bool 40 } 41 42 // DefaultOpt is the default values used for Options 43 var DefaultOpt = Options{ 44 Auth: libhttp.DefaultAuthCfg(), 45 HTTP: libhttp.DefaultCfg(), 46 } 47 48 // Opt is options set by command line flags 49 var Opt = DefaultOpt 50 51 // flagPrefix is the prefix used to uniquely identify command line flags. 52 // It is intentionally empty for this package. 53 const flagPrefix = "" 54 55 func init() { 56 flagSet := Command.Flags() 57 libhttp.AddAuthFlagsPrefix(flagSet, flagPrefix, &Opt.Auth) 58 libhttp.AddHTTPFlagsPrefix(flagSet, flagPrefix, &Opt.HTTP) 59 flags.BoolVarP(flagSet, &Opt.Stdio, "stdio", "", false, "Run an HTTP2 server on stdin/stdout", "") 60 flags.BoolVarP(flagSet, &Opt.AppendOnly, "append-only", "", false, "Disallow deletion of repository data", "") 61 flags.BoolVarP(flagSet, &Opt.PrivateRepos, "private-repos", "", false, "Users can only access their private repo", "") 62 flags.BoolVarP(flagSet, &Opt.CacheObjects, "cache-objects", "", true, "Cache listed objects", "") 63 } 64 65 // Command definition for cobra 66 var Command = &cobra.Command{ 67 Use: "restic remote:path", 68 Short: `Serve the remote for restic's REST API.`, 69 Long: `Run a basic web server to serve a remote over restic's REST backend 70 API over HTTP. This allows restic to use rclone as a data storage 71 mechanism for cloud providers that restic does not support directly. 72 73 [Restic](https://restic.net/) is a command-line program for doing 74 backups. 75 76 The server will log errors. Use -v to see access logs. 77 78 ` + "`--bwlimit`" + ` will be respected for file transfers. 79 Use ` + "`--stats`" + ` to control the stats printing. 80 81 ### Setting up rclone for use by restic ### 82 83 First [set up a remote for your chosen cloud provider](/docs/#configure). 84 85 Once you have set up the remote, check it is working with, for example 86 "rclone lsd remote:". You may have called the remote something other 87 than "remote:" - just substitute whatever you called it in the 88 following instructions. 89 90 Now start the rclone restic server 91 92 rclone serve restic -v remote:backup 93 94 Where you can replace "backup" in the above by whatever path in the 95 remote you wish to use. 96 97 By default this will serve on "localhost:8080" you can change this 98 with use of the ` + "`--addr`" + ` flag. 99 100 You might wish to start this server on boot. 101 102 Adding ` + "`--cache-objects=false`" + ` will cause rclone to stop caching objects 103 returned from the List call. Caching is normally desirable as it speeds 104 up downloading objects, saves transactions and uses very little memory. 105 106 ### Setting up restic to use rclone ### 107 108 Now you can [follow the restic 109 instructions](http://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server) 110 on setting up restic. 111 112 Note that you will need restic 0.8.2 or later to interoperate with 113 rclone. 114 115 For the example above you will want to use "http://localhost:8080/" as 116 the URL for the REST server. 117 118 For example: 119 120 $ export RESTIC_REPOSITORY=rest:http://localhost:8080/ 121 $ export RESTIC_PASSWORD=yourpassword 122 $ restic init 123 created restic backend 8b1a4b56ae at rest:http://localhost:8080/ 124 125 Please note that knowledge of your password is required to access 126 the repository. Losing your password means that your data is 127 irrecoverably lost. 128 $ restic backup /path/to/files/to/backup 129 scan [/path/to/files/to/backup] 130 scanned 189 directories, 312 files in 0:00 131 [0:00] 100.00% 38.128 MiB / 38.128 MiB 501 / 501 items 0 errors ETA 0:00 132 duration: 0:00 133 snapshot 45c8fdd8 saved 134 135 #### Multiple repositories #### 136 137 Note that you can use the endpoint to host multiple repositories. Do 138 this by adding a directory name or path after the URL. Note that 139 these **must** end with /. Eg 140 141 $ export RESTIC_REPOSITORY=rest:http://localhost:8080/user1repo/ 142 # backup user1 stuff 143 $ export RESTIC_REPOSITORY=rest:http://localhost:8080/user2repo/ 144 # backup user2 stuff 145 146 #### Private repositories #### 147 148 The` + "`--private-repos`" + ` flag can be used to limit users to repositories starting 149 with a path of ` + "`/<username>/`" + `. 150 151 ` + libhttp.Help(flagPrefix) + libhttp.AuthHelp(flagPrefix), 152 Annotations: map[string]string{ 153 "versionIntroduced": "v1.40", 154 }, 155 Run: func(command *cobra.Command, args []string) { 156 ctx := context.Background() 157 cmd.CheckArgs(1, 1, command, args) 158 f := cmd.NewFsSrc(args) 159 cmd.Run(false, true, command, func() error { 160 s, err := newServer(ctx, f, &Opt) 161 if err != nil { 162 return err 163 } 164 if s.opt.Stdio { 165 if terminal.IsTerminal(int(os.Stdout.Fd())) { 166 return errors.New("refusing to run HTTP2 server directly on a terminal, please let restic start rclone") 167 } 168 169 conn := &StdioConn{ 170 stdin: os.Stdin, 171 stdout: os.Stdout, 172 } 173 174 httpSrv := &http2.Server{} 175 opts := &http2.ServeConnOpts{ 176 Handler: s.Server.Router(), 177 } 178 httpSrv.ServeConn(conn, opts) 179 return nil 180 } 181 fs.Logf(s.f, "Serving restic REST API on %s", s.URLs()) 182 183 defer systemd.Notify()() 184 s.Wait() 185 186 return nil 187 }) 188 }, 189 } 190 191 const ( 192 resticAPIV2 = "application/vnd.x.restic.rest.v2" 193 ) 194 195 type contextRemoteType struct{} 196 197 // ContextRemoteKey is a simple context key for storing the username of the request 198 var ContextRemoteKey = &contextRemoteType{} 199 200 // WithRemote makes a remote from a URL path. This implements the backend layout 201 // required by restic. 202 func WithRemote(next http.Handler) http.Handler { 203 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 204 var urlpath string 205 rctx := chi.RouteContext(r.Context()) 206 if rctx != nil && rctx.RoutePath != "" { 207 urlpath = rctx.RoutePath 208 } else { 209 urlpath = r.URL.Path 210 } 211 urlpath = strings.Trim(urlpath, "/") 212 parts := matchData.FindStringSubmatch(urlpath) 213 // if no data directory, layout is flat 214 if parts != nil { 215 // otherwise map 216 // data/2159dd48 to 217 // data/21/2159dd48 218 fileName := parts[1] 219 prefix := urlpath[:len(urlpath)-len(fileName)] 220 urlpath = prefix + fileName[:2] + "/" + fileName 221 } 222 ctx := context.WithValue(r.Context(), ContextRemoteKey, urlpath) 223 next.ServeHTTP(w, r.WithContext(ctx)) 224 }) 225 } 226 227 // Middleware to ensure authenticated user is accessing their own private folder 228 func checkPrivate(next http.Handler) http.Handler { 229 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 230 user := chi.URLParam(r, "userID") 231 userID, ok := libhttp.CtxGetUser(r.Context()) 232 if ok && user != "" && user == userID { 233 next.ServeHTTP(w, r) 234 } else { 235 http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) 236 } 237 }) 238 } 239 240 // server contains everything to run the server 241 type server struct { 242 *libhttp.Server 243 f fs.Fs 244 cache *cache 245 opt Options 246 } 247 248 func newServer(ctx context.Context, f fs.Fs, opt *Options) (s *server, err error) { 249 s = &server{ 250 f: f, 251 cache: newCache(opt.CacheObjects), 252 opt: *opt, 253 } 254 // Don't bind any HTTP listeners if running with --stdio 255 if opt.Stdio { 256 opt.HTTP.ListenAddr = nil 257 } 258 s.Server, err = libhttp.NewServer(ctx, 259 libhttp.WithConfig(opt.HTTP), 260 libhttp.WithAuth(opt.Auth), 261 ) 262 if err != nil { 263 return nil, fmt.Errorf("failed to init server: %w", err) 264 } 265 router := s.Router() 266 s.Bind(router) 267 s.Server.Serve() 268 return s, nil 269 } 270 271 // bind helper for main Bind method 272 func (s *server) bind(router chi.Router) { 273 router.MethodFunc("GET", "/*", func(w http.ResponseWriter, r *http.Request) { 274 urlpath := chi.URLParam(r, "*") 275 if urlpath == "" || strings.HasSuffix(urlpath, "/") { 276 s.listObjects(w, r) 277 } else { 278 s.serveObject(w, r) 279 } 280 }) 281 router.MethodFunc("POST", "/*", func(w http.ResponseWriter, r *http.Request) { 282 urlpath := chi.URLParam(r, "*") 283 if urlpath == "" || strings.HasSuffix(urlpath, "/") { 284 s.createRepo(w, r) 285 } else { 286 s.postObject(w, r) 287 } 288 }) 289 router.MethodFunc("HEAD", "/*", s.serveObject) 290 router.MethodFunc("DELETE", "/*", s.deleteObject) 291 } 292 293 // Bind restic server routes to passed router 294 func (s *server) Bind(router chi.Router) { 295 // FIXME 296 // if m := authX.Auth(authX.Opt); m != nil { 297 // router.Use(m) 298 // } 299 router.Use( 300 middleware.SetHeader("Accept-Ranges", "bytes"), 301 middleware.SetHeader("Server", "rclone/"+fs.Version), 302 WithRemote, 303 ) 304 305 if s.opt.PrivateRepos { 306 router.Route("/{userID}", func(r chi.Router) { 307 r.Use(checkPrivate) 308 s.bind(r) 309 }) 310 router.NotFound(func(w http.ResponseWriter, _ *http.Request) { 311 http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) 312 }) 313 } else { 314 s.bind(router) 315 } 316 } 317 318 var matchData = regexp.MustCompile("(?:^|/)data/([^/]{2,})$") 319 320 // newObject returns an object with the remote given either from the 321 // cache or directly 322 func (s *server) newObject(ctx context.Context, remote string) (fs.Object, error) { 323 o := s.cache.find(remote) 324 if o != nil { 325 return o, nil 326 } 327 o, err := s.f.NewObject(ctx, remote) 328 if err != nil { 329 return o, err 330 } 331 s.cache.add(remote, o) 332 return o, nil 333 } 334 335 // get the remote 336 func (s *server) serveObject(w http.ResponseWriter, r *http.Request) { 337 remote, ok := r.Context().Value(ContextRemoteKey).(string) 338 if !ok { 339 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 340 return 341 } 342 o, err := s.newObject(r.Context(), remote) 343 if err != nil { 344 fs.Debugf(remote, "%s request error: %v", r.Method, err) 345 if errors.Is(err, fs.ErrorObjectNotFound) { 346 http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 347 } else { 348 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 349 } 350 return 351 } 352 serve.Object(w, r, o) 353 } 354 355 // postObject posts an object to the repository 356 func (s *server) postObject(w http.ResponseWriter, r *http.Request) { 357 remote, ok := r.Context().Value(ContextRemoteKey).(string) 358 if !ok { 359 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 360 return 361 } 362 if s.opt.AppendOnly { 363 // make sure the file does not exist yet 364 _, err := s.newObject(r.Context(), remote) 365 if err == nil { 366 fs.Errorf(remote, "Post request: file already exists, refusing to overwrite in append-only mode") 367 http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) 368 369 return 370 } 371 } 372 373 o, err := operations.RcatSize(r.Context(), s.f, remote, r.Body, r.ContentLength, time.Now(), nil) 374 if err != nil { 375 err = accounting.Stats(r.Context()).Error(err) 376 fs.Errorf(remote, "Post request rcat error: %v", err) 377 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 378 379 return 380 } 381 382 // if successfully uploaded add to cache 383 s.cache.add(remote, o) 384 } 385 386 // delete the remote 387 func (s *server) deleteObject(w http.ResponseWriter, r *http.Request) { 388 remote, ok := r.Context().Value(ContextRemoteKey).(string) 389 if !ok { 390 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 391 return 392 } 393 if s.opt.AppendOnly { 394 parts := strings.Split(r.URL.Path, "/") 395 396 // if path doesn't end in "/locks/:name", disallow the operation 397 if len(parts) < 2 || parts[len(parts)-2] != "locks" { 398 http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) 399 return 400 } 401 } 402 403 o, err := s.newObject(r.Context(), remote) 404 if err != nil { 405 fs.Debugf(remote, "Delete request error: %v", err) 406 http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 407 return 408 } 409 410 if err := o.Remove(r.Context()); err != nil { 411 fs.Errorf(remote, "Delete request remove error: %v", err) 412 if errors.Is(err, fs.ErrorObjectNotFound) { 413 http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) 414 } else { 415 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 416 } 417 return 418 } 419 420 // remove object from cache 421 s.cache.remove(remote) 422 } 423 424 // listItem is an element returned for the restic v2 list response 425 type listItem struct { 426 Name string `json:"name"` 427 Size int64 `json:"size"` 428 } 429 430 // return type for list 431 type listItems []listItem 432 433 // add an fs.Object to the listItems 434 func (ls *listItems) add(o fs.Object) { 435 *ls = append(*ls, listItem{ 436 Name: path.Base(o.Remote()), 437 Size: o.Size(), 438 }) 439 } 440 441 // listObjects lists all Objects of a given type in an arbitrary order. 442 func (s *server) listObjects(w http.ResponseWriter, r *http.Request) { 443 remote, ok := r.Context().Value(ContextRemoteKey).(string) 444 if !ok { 445 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 446 return 447 } 448 if r.Header.Get("Accept") != resticAPIV2 { 449 fs.Errorf(remote, "Restic v2 API required for List Objects") 450 http.Error(w, "Restic v2 API required for List Objects", http.StatusBadRequest) 451 return 452 } 453 fs.Debugf(remote, "list request") 454 455 // make sure an empty list is returned, and not a 'nil' value 456 ls := listItems{} 457 458 // Remove all existing values from the cache 459 s.cache.removePrefix(remote) 460 461 // if remote supports ListR use that directly, otherwise use recursive Walk 462 err := walk.ListR(r.Context(), s.f, remote, true, -1, walk.ListObjects, func(entries fs.DirEntries) error { 463 for _, entry := range entries { 464 if o, ok := entry.(fs.Object); ok { 465 ls.add(o) 466 s.cache.add(o.Remote(), o) 467 } 468 } 469 return nil 470 }) 471 if err != nil { 472 if !errors.Is(err, fs.ErrorDirNotFound) { 473 fs.Errorf(remote, "list failed: %#v %T", err, err) 474 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 475 return 476 } 477 } 478 479 w.Header().Set("Content-Type", "application/vnd.x.restic.rest.v2") 480 enc := json.NewEncoder(w) 481 err = enc.Encode(ls) 482 if err != nil { 483 fs.Errorf(remote, "failed to write list: %v", err) 484 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 485 return 486 } 487 } 488 489 // createRepo creates repository directories. 490 // 491 // We don't bother creating the data dirs as rclone will create them on the fly 492 func (s *server) createRepo(w http.ResponseWriter, r *http.Request) { 493 remote, ok := r.Context().Value(ContextRemoteKey).(string) 494 if !ok { 495 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 496 return 497 } 498 fs.Infof(remote, "Creating repository") 499 500 if r.URL.Query().Get("create") != "true" { 501 http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 502 return 503 } 504 505 err := s.f.Mkdir(r.Context(), remote) 506 if err != nil { 507 fs.Errorf(remote, "Create repo failed to Mkdir: %v", err) 508 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 509 return 510 } 511 512 for _, name := range []string{"data", "index", "keys", "locks", "snapshots"} { 513 dirRemote := path.Join(remote, name) 514 err := s.f.Mkdir(r.Context(), dirRemote) 515 if err != nil { 516 fs.Errorf(dirRemote, "Create repo failed to Mkdir: %v", err) 517 http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) 518 return 519 } 520 } 521 }