github.com/divyam234/rclone@v1.64.1/fs/newfs.go (about) 1 // NewFs and its helpers 2 3 package fs 4 5 import ( 6 "context" 7 "crypto/md5" 8 "encoding/base64" 9 "os" 10 "path/filepath" 11 "strings" 12 "sync" 13 14 "github.com/divyam234/rclone/fs/config/configmap" 15 "github.com/divyam234/rclone/fs/fspath" 16 ) 17 18 // Store the hashes of the overridden config 19 var ( 20 overriddenConfigMu sync.Mutex 21 overriddenConfig = make(map[string]string) 22 ) 23 24 // NewFs makes a new Fs object from the path 25 // 26 // The path is of the form remote:path 27 // 28 // Remotes are looked up in the config file. If the remote isn't 29 // found then NotFoundInConfigFile will be returned. 30 // 31 // On Windows avoid single character remote names as they can be mixed 32 // up with drive letters. 33 func NewFs(ctx context.Context, path string) (Fs, error) { 34 Debugf(nil, "Creating backend with remote %q", path) 35 if ConfigFileHasSection(path) { 36 Logf(nil, "%q refers to a local folder, use %q to refer to your remote or %q to hide this warning", path, path+":", "./"+path) 37 } 38 fsInfo, configName, fsPath, config, err := ConfigFs(path) 39 if err != nil { 40 return nil, err 41 } 42 overridden := fsInfo.Options.Overridden(config) 43 if len(overridden) > 0 { 44 extraConfig := overridden.String() 45 //Debugf(nil, "detected overridden config %q", extraConfig) 46 md5sumBinary := md5.Sum([]byte(extraConfig)) 47 configHash := base64.RawURLEncoding.EncodeToString(md5sumBinary[:]) 48 // 5 characters length is 5*6 = 30 bits of base64 49 overriddenConfigMu.Lock() 50 var suffix string 51 for maxLength := 5; ; maxLength++ { 52 suffix = "{" + configHash[:maxLength] + "}" 53 existingExtraConfig, ok := overriddenConfig[suffix] 54 if !ok || existingExtraConfig == extraConfig { 55 break 56 } 57 } 58 Debugf(configName, "detected overridden config - adding %q suffix to name", suffix) 59 // Add the suffix to the config name 60 // 61 // These need to work as filesystem names as the VFS cache will use them 62 configName += suffix 63 // Store the config suffixes for reversing in ConfigString 64 overriddenConfig[suffix] = extraConfig 65 overriddenConfigMu.Unlock() 66 } 67 f, err := fsInfo.NewFs(ctx, configName, fsPath, config) 68 if f != nil && (err == nil || err == ErrorIsFile) { 69 addReverse(f, fsInfo) 70 } 71 return f, err 72 } 73 74 // ConfigFs makes the config for calling NewFs with. 75 // 76 // It parses the path which is of the form remote:path 77 // 78 // Remotes are looked up in the config file. If the remote isn't 79 // found then NotFoundInConfigFile will be returned. 80 func ConfigFs(path string) (fsInfo *RegInfo, configName, fsPath string, config *configmap.Map, err error) { 81 // Parse the remote path 82 fsInfo, configName, fsPath, connectionStringConfig, err := ParseRemote(path) 83 if err != nil { 84 return 85 } 86 config = ConfigMap(fsInfo, configName, connectionStringConfig) 87 return 88 } 89 90 // ParseRemote deconstructs a path into configName, fsPath, looking up 91 // the fsName in the config file (returning NotFoundInConfigFile if not found) 92 func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, connectionStringConfig configmap.Simple, err error) { 93 parsed, err := fspath.Parse(path) 94 if err != nil { 95 return nil, "", "", nil, err 96 } 97 configName, fsPath = parsed.Name, parsed.Path 98 var fsName string 99 var ok bool 100 if configName != "" { 101 if strings.HasPrefix(configName, ":") { 102 fsName = configName[1:] 103 } else { 104 m := ConfigMap(nil, configName, parsed.Config) 105 fsName, ok = m.Get("type") 106 if !ok { 107 return nil, "", "", nil, ErrorNotFoundInConfigFile 108 } 109 } 110 } else { 111 fsName = "local" 112 configName = "local" 113 } 114 fsInfo, err = Find(fsName) 115 return fsInfo, configName, fsPath, parsed.Config, err 116 } 117 118 // configString returns a canonical version of the config string used 119 // to configure the Fs as passed to fs.NewFs 120 func configString(f Fs, full bool) string { 121 name := f.Name() 122 if open := strings.IndexRune(name, '{'); full && open >= 0 && strings.HasSuffix(name, "}") { 123 suffix := name[open:] 124 overriddenConfigMu.Lock() 125 config, ok := overriddenConfig[suffix] 126 overriddenConfigMu.Unlock() 127 if ok { 128 name = name[:open] + "," + config 129 } else { 130 Errorf(f, "Failed to find config for suffix %q", suffix) 131 } 132 } 133 root := f.Root() 134 if name == "local" && f.Features().IsLocal { 135 return root 136 } 137 return name + ":" + root 138 } 139 140 // ConfigString returns a canonical version of the config string used 141 // to configure the Fs as passed to fs.NewFs. For Fs with extra 142 // parameters this will include a canonical {hexstring} suffix. 143 func ConfigString(f Fs) string { 144 return configString(f, false) 145 } 146 147 // ConfigStringFull returns a canonical version of the config string 148 // used to configure the Fs as passed to fs.NewFs. This string can be 149 // used to re-instantiate the Fs exactly so includes all the extra 150 // parameters passed in. 151 func ConfigStringFull(f Fs) string { 152 return configString(f, true) 153 } 154 155 // TemporaryLocalFs creates a local FS in the OS's temporary directory. 156 // 157 // No cleanup is performed, the caller must call Purge on the Fs themselves. 158 func TemporaryLocalFs(ctx context.Context) (Fs, error) { 159 path, err := os.MkdirTemp("", "rclone-spool") 160 if err == nil { 161 err = os.Remove(path) 162 } 163 if err != nil { 164 return nil, err 165 } 166 path = filepath.ToSlash(path) 167 return NewFs(ctx, path) 168 }