github.com/divyam234/rclone@v1.64.1/cmd/serve/proxy/proxy.go (about)

     1  // Package proxy implements a programmable proxy for rclone serve
     2  package proxy
     3  
     4  import (
     5  	"bytes"
     6  	"context"
     7  	"crypto/sha256"
     8  	"crypto/subtle"
     9  	"encoding/json"
    10  	"errors"
    11  	"fmt"
    12  	"os/exec"
    13  	"strings"
    14  	"time"
    15  
    16  	"github.com/divyam234/rclone/fs"
    17  	"github.com/divyam234/rclone/fs/cache"
    18  	"github.com/divyam234/rclone/fs/config/configmap"
    19  	"github.com/divyam234/rclone/fs/config/obscure"
    20  	libcache "github.com/divyam234/rclone/lib/cache"
    21  	"github.com/divyam234/rclone/vfs"
    22  	"github.com/divyam234/rclone/vfs/vfsflags"
    23  )
    24  
    25  // Help contains text describing how to use the proxy
    26  var Help = strings.Replace(`
    27  ### Auth Proxy
    28  
    29  If you supply the parameter |--auth-proxy /path/to/program| then
    30  rclone will use that program to generate backends on the fly which
    31  then are used to authenticate incoming requests.  This uses a simple
    32  JSON based protocol with input on STDIN and output on STDOUT.
    33  
    34  **PLEASE NOTE:** |--auth-proxy| and |--authorized-keys| cannot be used
    35  together, if |--auth-proxy| is set the authorized keys option will be
    36  ignored.
    37  
    38  There is an example program
    39  [bin/test_proxy.py](https://github.com/divyam234/rclone/blob/master/test_proxy.py)
    40  in the rclone source code.
    41  
    42  The program's job is to take a |user| and |pass| on the input and turn
    43  those into the config for a backend on STDOUT in JSON format.  This
    44  config will have any default parameters for the backend added, but it
    45  won't use configuration from environment variables or command line
    46  options - it is the job of the proxy program to make a complete
    47  config.
    48  
    49  This config generated must have this extra parameter
    50  - |_root| - root to use for the backend
    51  
    52  And it may have this parameter
    53  - |_obscure| - comma separated strings for parameters to obscure
    54  
    55  If password authentication was used by the client, input to the proxy
    56  process (on STDIN) would look similar to this:
    57  
    58  |||
    59  {
    60  	"user": "me",
    61  	"pass": "mypassword"
    62  }
    63  |||
    64  
    65  If public-key authentication was used by the client, input to the
    66  proxy process (on STDIN) would look similar to this:
    67  
    68  |||
    69  {
    70  	"user": "me",
    71  	"public_key": "AAAAB3NzaC1yc2EAAAADAQABAAABAQDuwESFdAe14hVS6omeyX7edc...JQdf"
    72  }
    73  |||
    74  
    75  And as an example return this on STDOUT
    76  
    77  |||
    78  {
    79  	"type": "sftp",
    80  	"_root": "",
    81  	"_obscure": "pass",
    82  	"user": "me",
    83  	"pass": "mypassword",
    84  	"host": "sftp.example.com"
    85  }
    86  |||
    87  
    88  This would mean that an SFTP backend would be created on the fly for
    89  the |user| and |pass|/|public_key| returned in the output to the host given.  Note
    90  that since |_obscure| is set to |pass|, rclone will obscure the |pass|
    91  parameter before creating the backend (which is required for sftp
    92  backends).
    93  
    94  The program can manipulate the supplied |user| in any way, for example
    95  to make proxy to many different sftp backends, you could make the
    96  |user| be |user@example.com| and then set the |host| to |example.com|
    97  in the output and the user to |user|. For security you'd probably want
    98  to restrict the |host| to a limited list.
    99  
   100  Note that an internal cache is keyed on |user| so only use that for
   101  configuration, don't use |pass| or |public_key|.  This also means that if a user's
   102  password or public-key is changed the cache will need to expire (which takes 5 mins)
   103  before it takes effect.
   104  
   105  This can be used to build general purpose proxies to any kind of
   106  backend that rclone supports.  
   107  `, "|", "`", -1)
   108  
   109  // Options is options for creating the proxy
   110  type Options struct {
   111  	AuthProxy string
   112  }
   113  
   114  // DefaultOpt is the default values uses for Opt
   115  var DefaultOpt = Options{
   116  	AuthProxy: "",
   117  }
   118  
   119  // Proxy represents a proxy to turn auth requests into a VFS
   120  type Proxy struct {
   121  	cmdLine  []string // broken down command line
   122  	vfsCache *libcache.Cache
   123  	ctx      context.Context // for global config
   124  	Opt      Options
   125  }
   126  
   127  // cacheEntry is what is stored in the vfsCache
   128  type cacheEntry struct {
   129  	vfs    *vfs.VFS          // stored VFS
   130  	pwHash [sha256.Size]byte // sha256 hash of the password/publicKey
   131  }
   132  
   133  // New creates a new proxy with the Options passed in
   134  func New(ctx context.Context, opt *Options) *Proxy {
   135  	return &Proxy{
   136  		ctx:      ctx,
   137  		Opt:      *opt,
   138  		cmdLine:  strings.Fields(opt.AuthProxy),
   139  		vfsCache: libcache.New(),
   140  	}
   141  }
   142  
   143  // run the proxy command returning a config map
   144  func (p *Proxy) run(in map[string]string) (config configmap.Simple, err error) {
   145  	cmd := exec.Command(p.cmdLine[0], p.cmdLine[1:]...)
   146  	inBytes, err := json.MarshalIndent(in, "", "\t")
   147  	if err != nil {
   148  		return nil, fmt.Errorf("proxy: failed to marshal input: %w", err)
   149  	}
   150  	var stdout, stderr bytes.Buffer
   151  	cmd.Stdin = bytes.NewBuffer(inBytes)
   152  	cmd.Stdout = &stdout
   153  	cmd.Stderr = &stderr
   154  	start := time.Now()
   155  	err = cmd.Run()
   156  	fs.Debugf(nil, "Calling proxy %v", p.cmdLine)
   157  	duration := time.Since(start)
   158  	if err != nil {
   159  		return nil, fmt.Errorf("proxy: failed on %v: %q: %w", p.cmdLine, strings.TrimSpace(stderr.String()), err)
   160  	}
   161  	err = json.Unmarshal(stdout.Bytes(), &config)
   162  	if err != nil {
   163  		return nil, fmt.Errorf("proxy: failed to read output: %q: %w", stdout.String(), err)
   164  	}
   165  	fs.Debugf(nil, "Proxy returned in %v", duration)
   166  
   167  	// Obscure any values in the config map that need it
   168  	obscureFields, ok := config.Get("_obscure")
   169  	if ok {
   170  		for _, key := range strings.Split(obscureFields, ",") {
   171  			value, ok := config.Get(key)
   172  			if ok {
   173  				obscuredValue, err := obscure.Obscure(value)
   174  				if err != nil {
   175  					return nil, fmt.Errorf("proxy: %w", err)
   176  				}
   177  				config.Set(key, obscuredValue)
   178  			}
   179  		}
   180  	}
   181  	return config, nil
   182  }
   183  
   184  // call runs the auth proxy and returns a cacheEntry and an error
   185  func (p *Proxy) call(user, auth string, isPublicKey bool) (value interface{}, err error) {
   186  	var config configmap.Simple
   187  	// Contact the proxy
   188  	if isPublicKey {
   189  		config, err = p.run(map[string]string{
   190  			"user":       user,
   191  			"public_key": auth,
   192  		})
   193  	} else {
   194  		config, err = p.run(map[string]string{
   195  			"user": user,
   196  			"pass": auth,
   197  		})
   198  	}
   199  
   200  	if err != nil {
   201  		return nil, err
   202  	}
   203  
   204  	// Look for required fields in the answer
   205  	fsName, ok := config.Get("type")
   206  	if !ok {
   207  		return nil, errors.New("proxy: type not set in result")
   208  	}
   209  	root, ok := config.Get("_root")
   210  	if !ok {
   211  		return nil, errors.New("proxy: _root not set in result")
   212  	}
   213  
   214  	// Find the backend
   215  	fsInfo, err := fs.Find(fsName)
   216  	if err != nil {
   217  		return nil, fmt.Errorf("proxy: couldn't find backend for %q: %w", fsName, err)
   218  	}
   219  
   220  	// base name of config on user name.  This may appear in logs
   221  	name := "proxy-" + user
   222  	fsString := name + ":" + root
   223  
   224  	// Look for fs in the VFS cache
   225  	value, err = p.vfsCache.Get(user, func(key string) (value interface{}, ok bool, err error) {
   226  		// Create the Fs from the cache
   227  		f, err := cache.GetFn(p.ctx, fsString, func(ctx context.Context, fsString string) (fs.Fs, error) {
   228  			// Update the config with the default values
   229  			for i := range fsInfo.Options {
   230  				o := &fsInfo.Options[i]
   231  				if _, found := config.Get(o.Name); !found && o.Default != nil && o.String() != "" {
   232  					config.Set(o.Name, o.String())
   233  				}
   234  			}
   235  			return fsInfo.NewFs(ctx, name, root, config)
   236  		})
   237  		if err != nil {
   238  			return nil, false, err
   239  		}
   240  
   241  		// We hash the auth here so we don't copy the auth more than we
   242  		// need to in memory. An attacker would find it easier to go
   243  		// after the unencrypted password in memory most likely.
   244  		entry := cacheEntry{
   245  			vfs:    vfs.New(f, &vfsflags.Opt),
   246  			pwHash: sha256.Sum256([]byte(auth)),
   247  		}
   248  		return entry, true, nil
   249  	})
   250  	if err != nil {
   251  		return nil, fmt.Errorf("proxy: failed to create backend: %w", err)
   252  	}
   253  	return value, nil
   254  }
   255  
   256  // Call runs the auth proxy with the username and password/public key provided
   257  // returning a *vfs.VFS and the key used in the VFS cache.
   258  func (p *Proxy) Call(user, auth string, isPublicKey bool) (VFS *vfs.VFS, vfsKey string, err error) {
   259  	// Look in the cache first
   260  	value, ok := p.vfsCache.GetMaybe(user)
   261  
   262  	// If not found then call the proxy for a fresh answer
   263  	if !ok {
   264  		value, err = p.call(user, auth, isPublicKey)
   265  		if err != nil {
   266  			return nil, "", err
   267  		}
   268  	}
   269  
   270  	// check we got what we were expecting
   271  	entry, ok := value.(cacheEntry)
   272  	if !ok {
   273  		return nil, "", fmt.Errorf("proxy: value is not cache entry: %#v", value)
   274  	}
   275  
   276  	// Check the password / public key is correct in the cached entry.  This
   277  	// prevents an attack where subsequent requests for the same
   278  	// user don't have their auth checked. It does mean that if
   279  	// the password is changed, the user will have to wait for
   280  	// cache expiry (5m) before trying again.
   281  	authHash := sha256.Sum256([]byte(auth))
   282  	if subtle.ConstantTimeCompare(authHash[:], entry.pwHash[:]) != 1 {
   283  		if isPublicKey {
   284  			return nil, "", errors.New("proxy: incorrect public key")
   285  		}
   286  		return nil, "", errors.New("proxy: incorrect password")
   287  	}
   288  
   289  	return entry.vfs, user, nil
   290  }
   291  
   292  // Get VFS from the cache using key - returns nil if not found
   293  func (p *Proxy) Get(key string) *vfs.VFS {
   294  	value, ok := p.vfsCache.GetMaybe(key)
   295  	if !ok {
   296  		return nil
   297  	}
   298  	entry := value.(cacheEntry)
   299  	return entry.vfs
   300  }