github.com/divyam234/rclone@v1.64.1/cmd/serve/proxy/proxy.go (about) 1 // Package proxy implements a programmable proxy for rclone serve 2 package proxy 3 4 import ( 5 "bytes" 6 "context" 7 "crypto/sha256" 8 "crypto/subtle" 9 "encoding/json" 10 "errors" 11 "fmt" 12 "os/exec" 13 "strings" 14 "time" 15 16 "github.com/divyam234/rclone/fs" 17 "github.com/divyam234/rclone/fs/cache" 18 "github.com/divyam234/rclone/fs/config/configmap" 19 "github.com/divyam234/rclone/fs/config/obscure" 20 libcache "github.com/divyam234/rclone/lib/cache" 21 "github.com/divyam234/rclone/vfs" 22 "github.com/divyam234/rclone/vfs/vfsflags" 23 ) 24 25 // Help contains text describing how to use the proxy 26 var Help = strings.Replace(` 27 ### Auth Proxy 28 29 If you supply the parameter |--auth-proxy /path/to/program| then 30 rclone will use that program to generate backends on the fly which 31 then are used to authenticate incoming requests. This uses a simple 32 JSON based protocol with input on STDIN and output on STDOUT. 33 34 **PLEASE NOTE:** |--auth-proxy| and |--authorized-keys| cannot be used 35 together, if |--auth-proxy| is set the authorized keys option will be 36 ignored. 37 38 There is an example program 39 [bin/test_proxy.py](https://github.com/divyam234/rclone/blob/master/test_proxy.py) 40 in the rclone source code. 41 42 The program's job is to take a |user| and |pass| on the input and turn 43 those into the config for a backend on STDOUT in JSON format. This 44 config will have any default parameters for the backend added, but it 45 won't use configuration from environment variables or command line 46 options - it is the job of the proxy program to make a complete 47 config. 48 49 This config generated must have this extra parameter 50 - |_root| - root to use for the backend 51 52 And it may have this parameter 53 - |_obscure| - comma separated strings for parameters to obscure 54 55 If password authentication was used by the client, input to the proxy 56 process (on STDIN) would look similar to this: 57 58 ||| 59 { 60 "user": "me", 61 "pass": "mypassword" 62 } 63 ||| 64 65 If public-key authentication was used by the client, input to the 66 proxy process (on STDIN) would look similar to this: 67 68 ||| 69 { 70 "user": "me", 71 "public_key": "AAAAB3NzaC1yc2EAAAADAQABAAABAQDuwESFdAe14hVS6omeyX7edc...JQdf" 72 } 73 ||| 74 75 And as an example return this on STDOUT 76 77 ||| 78 { 79 "type": "sftp", 80 "_root": "", 81 "_obscure": "pass", 82 "user": "me", 83 "pass": "mypassword", 84 "host": "sftp.example.com" 85 } 86 ||| 87 88 This would mean that an SFTP backend would be created on the fly for 89 the |user| and |pass|/|public_key| returned in the output to the host given. Note 90 that since |_obscure| is set to |pass|, rclone will obscure the |pass| 91 parameter before creating the backend (which is required for sftp 92 backends). 93 94 The program can manipulate the supplied |user| in any way, for example 95 to make proxy to many different sftp backends, you could make the 96 |user| be |user@example.com| and then set the |host| to |example.com| 97 in the output and the user to |user|. For security you'd probably want 98 to restrict the |host| to a limited list. 99 100 Note that an internal cache is keyed on |user| so only use that for 101 configuration, don't use |pass| or |public_key|. This also means that if a user's 102 password or public-key is changed the cache will need to expire (which takes 5 mins) 103 before it takes effect. 104 105 This can be used to build general purpose proxies to any kind of 106 backend that rclone supports. 107 `, "|", "`", -1) 108 109 // Options is options for creating the proxy 110 type Options struct { 111 AuthProxy string 112 } 113 114 // DefaultOpt is the default values uses for Opt 115 var DefaultOpt = Options{ 116 AuthProxy: "", 117 } 118 119 // Proxy represents a proxy to turn auth requests into a VFS 120 type Proxy struct { 121 cmdLine []string // broken down command line 122 vfsCache *libcache.Cache 123 ctx context.Context // for global config 124 Opt Options 125 } 126 127 // cacheEntry is what is stored in the vfsCache 128 type cacheEntry struct { 129 vfs *vfs.VFS // stored VFS 130 pwHash [sha256.Size]byte // sha256 hash of the password/publicKey 131 } 132 133 // New creates a new proxy with the Options passed in 134 func New(ctx context.Context, opt *Options) *Proxy { 135 return &Proxy{ 136 ctx: ctx, 137 Opt: *opt, 138 cmdLine: strings.Fields(opt.AuthProxy), 139 vfsCache: libcache.New(), 140 } 141 } 142 143 // run the proxy command returning a config map 144 func (p *Proxy) run(in map[string]string) (config configmap.Simple, err error) { 145 cmd := exec.Command(p.cmdLine[0], p.cmdLine[1:]...) 146 inBytes, err := json.MarshalIndent(in, "", "\t") 147 if err != nil { 148 return nil, fmt.Errorf("proxy: failed to marshal input: %w", err) 149 } 150 var stdout, stderr bytes.Buffer 151 cmd.Stdin = bytes.NewBuffer(inBytes) 152 cmd.Stdout = &stdout 153 cmd.Stderr = &stderr 154 start := time.Now() 155 err = cmd.Run() 156 fs.Debugf(nil, "Calling proxy %v", p.cmdLine) 157 duration := time.Since(start) 158 if err != nil { 159 return nil, fmt.Errorf("proxy: failed on %v: %q: %w", p.cmdLine, strings.TrimSpace(stderr.String()), err) 160 } 161 err = json.Unmarshal(stdout.Bytes(), &config) 162 if err != nil { 163 return nil, fmt.Errorf("proxy: failed to read output: %q: %w", stdout.String(), err) 164 } 165 fs.Debugf(nil, "Proxy returned in %v", duration) 166 167 // Obscure any values in the config map that need it 168 obscureFields, ok := config.Get("_obscure") 169 if ok { 170 for _, key := range strings.Split(obscureFields, ",") { 171 value, ok := config.Get(key) 172 if ok { 173 obscuredValue, err := obscure.Obscure(value) 174 if err != nil { 175 return nil, fmt.Errorf("proxy: %w", err) 176 } 177 config.Set(key, obscuredValue) 178 } 179 } 180 } 181 return config, nil 182 } 183 184 // call runs the auth proxy and returns a cacheEntry and an error 185 func (p *Proxy) call(user, auth string, isPublicKey bool) (value interface{}, err error) { 186 var config configmap.Simple 187 // Contact the proxy 188 if isPublicKey { 189 config, err = p.run(map[string]string{ 190 "user": user, 191 "public_key": auth, 192 }) 193 } else { 194 config, err = p.run(map[string]string{ 195 "user": user, 196 "pass": auth, 197 }) 198 } 199 200 if err != nil { 201 return nil, err 202 } 203 204 // Look for required fields in the answer 205 fsName, ok := config.Get("type") 206 if !ok { 207 return nil, errors.New("proxy: type not set in result") 208 } 209 root, ok := config.Get("_root") 210 if !ok { 211 return nil, errors.New("proxy: _root not set in result") 212 } 213 214 // Find the backend 215 fsInfo, err := fs.Find(fsName) 216 if err != nil { 217 return nil, fmt.Errorf("proxy: couldn't find backend for %q: %w", fsName, err) 218 } 219 220 // base name of config on user name. This may appear in logs 221 name := "proxy-" + user 222 fsString := name + ":" + root 223 224 // Look for fs in the VFS cache 225 value, err = p.vfsCache.Get(user, func(key string) (value interface{}, ok bool, err error) { 226 // Create the Fs from the cache 227 f, err := cache.GetFn(p.ctx, fsString, func(ctx context.Context, fsString string) (fs.Fs, error) { 228 // Update the config with the default values 229 for i := range fsInfo.Options { 230 o := &fsInfo.Options[i] 231 if _, found := config.Get(o.Name); !found && o.Default != nil && o.String() != "" { 232 config.Set(o.Name, o.String()) 233 } 234 } 235 return fsInfo.NewFs(ctx, name, root, config) 236 }) 237 if err != nil { 238 return nil, false, err 239 } 240 241 // We hash the auth here so we don't copy the auth more than we 242 // need to in memory. An attacker would find it easier to go 243 // after the unencrypted password in memory most likely. 244 entry := cacheEntry{ 245 vfs: vfs.New(f, &vfsflags.Opt), 246 pwHash: sha256.Sum256([]byte(auth)), 247 } 248 return entry, true, nil 249 }) 250 if err != nil { 251 return nil, fmt.Errorf("proxy: failed to create backend: %w", err) 252 } 253 return value, nil 254 } 255 256 // Call runs the auth proxy with the username and password/public key provided 257 // returning a *vfs.VFS and the key used in the VFS cache. 258 func (p *Proxy) Call(user, auth string, isPublicKey bool) (VFS *vfs.VFS, vfsKey string, err error) { 259 // Look in the cache first 260 value, ok := p.vfsCache.GetMaybe(user) 261 262 // If not found then call the proxy for a fresh answer 263 if !ok { 264 value, err = p.call(user, auth, isPublicKey) 265 if err != nil { 266 return nil, "", err 267 } 268 } 269 270 // check we got what we were expecting 271 entry, ok := value.(cacheEntry) 272 if !ok { 273 return nil, "", fmt.Errorf("proxy: value is not cache entry: %#v", value) 274 } 275 276 // Check the password / public key is correct in the cached entry. This 277 // prevents an attack where subsequent requests for the same 278 // user don't have their auth checked. It does mean that if 279 // the password is changed, the user will have to wait for 280 // cache expiry (5m) before trying again. 281 authHash := sha256.Sum256([]byte(auth)) 282 if subtle.ConstantTimeCompare(authHash[:], entry.pwHash[:]) != 1 { 283 if isPublicKey { 284 return nil, "", errors.New("proxy: incorrect public key") 285 } 286 return nil, "", errors.New("proxy: incorrect password") 287 } 288 289 return entry.vfs, user, nil 290 } 291 292 // Get VFS from the cache using key - returns nil if not found 293 func (p *Proxy) Get(key string) *vfs.VFS { 294 value, ok := p.vfsCache.GetMaybe(key) 295 if !ok { 296 return nil 297 } 298 entry := value.(cacheEntry) 299 return entry.vfs 300 }