github.com/viant/toolbox@v0.34.5/storage/mem_service.go (about) 1 package storage 2 3 import ( 4 "bytes" 5 "errors" 6 "io" 7 "io/ioutil" 8 "net/url" 9 "os" 10 "strings" 11 "sync" 12 "time" 13 ) 14 15 const MemoryProviderScheme = "mem" 16 17 var noSuchFileOrDirectoryError = errors.New("No such file or directory") 18 var folderMode, _ = NewFileMode("drwxrwxrwx") 19 20 //MemoryRoot represents memory root storage 21 var MemoryRoot = newMemoryFolder("mem:///", NewFileInfo("/", 102, folderMode, time.Now(), true)) 22 23 //ResetMemory reset memory root storage 24 func ResetMemory() { 25 MemoryRoot = newMemoryFolder("mem:///", NewFileInfo("/", 102, folderMode, time.Now(), true)) 26 } 27 28 //Service represents memory storage service intended for testing 29 type memoryStorageService struct { 30 root *MemoryFolder 31 } 32 33 type MemoryFile struct { 34 name string 35 fileInfo os.FileInfo 36 content []byte 37 } 38 39 func (f *MemoryFile) Object() Object { 40 return NewAbstractStorageObject(f.name, f, f.fileInfo) 41 } 42 43 type MemoryFolder struct { 44 name string 45 fileInfo os.FileInfo 46 mutext *sync.RWMutex 47 files map[string]*MemoryFile 48 folders map[string]*MemoryFolder 49 } 50 51 func (f *MemoryFolder) Object() Object { 52 return NewAbstractStorageObject(f.name, f, f.fileInfo) 53 } 54 55 func (f *MemoryFolder) Objects() []Object { 56 var result = make([]Object, 0) 57 result = append(result, f.Object()) 58 for _, folder := range f.folders { 59 result = append(result, folder.Object()) 60 } 61 for _, file := range f.files { 62 result = append(result, file.Object()) 63 } 64 return result 65 } 66 67 func newMemoryFolder(name string, info os.FileInfo) *MemoryFolder { 68 return &MemoryFolder{ 69 name: name, 70 fileInfo: info, 71 mutext: &sync.RWMutex{}, 72 files: make(map[string]*MemoryFile), 73 folders: make(map[string]*MemoryFolder), 74 } 75 } 76 77 func (s *memoryStorageService) getFolder(pathFragments []string) (*MemoryFolder, error) { 78 node := s.root 79 var ok bool 80 for i := 1; i+1 < len(pathFragments); i++ { 81 pathFragment := pathFragments[i] 82 node, ok = node.folders[pathFragment] 83 if !ok { 84 return nil, noSuchFileOrDirectoryError 85 } 86 } 87 return node, nil 88 } 89 90 func (s *memoryStorageService) getPath(URL string) (string, error) { 91 parsedURL, err := url.Parse(URL) 92 if err != nil { 93 return "", err 94 } 95 parsedPath := parsedURL.Path 96 strings.Replace(parsedPath, "//", "/", len(parsedPath)) 97 if len(parsedPath) > 1 && strings.HasSuffix(parsedPath, "/") { 98 parsedPath = string(parsedPath[:len(parsedPath)-1]) 99 } 100 return parsedPath, nil 101 } 102 103 //List returns a list of object for supplied url 104 func (s *memoryStorageService) List(URL string) ([]Object, error) { 105 path, err := s.getPath(URL) 106 if err != nil { 107 return nil, err 108 } 109 if path == "/" { 110 return s.root.Objects(), nil 111 } 112 var pathFragments = strings.Split(path, "/") 113 node, err := s.getFolder(pathFragments) 114 if err != nil { 115 return nil, err 116 } 117 var pathLeaf = pathFragments[len(pathFragments)-1] 118 119 if memoryFile, ok := node.files[pathLeaf]; ok { 120 return []Object{memoryFile.Object()}, nil 121 } 122 if folder, ok := node.folders[pathLeaf]; ok { 123 return folder.Objects(), nil 124 } 125 126 return []Object{}, nil 127 } 128 129 //Exists returns true if resource exists 130 func (s *memoryStorageService) Exists(URL string) (bool, error) { 131 objects, err := s.List(URL) 132 if err != nil { 133 return false, err 134 } 135 return len(objects) > 0, nil 136 } 137 138 func (s *memoryStorageService) Close() error { 139 return nil 140 } 141 142 //Object returns a Object for supplied url 143 func (s *memoryStorageService) StorageObject(URL string) (Object, error) { 144 objects, err := s.List(URL) 145 if err != nil { 146 return nil, err 147 } 148 if len(objects) == 0 { 149 return nil, noSuchFileOrDirectoryError 150 } 151 return objects[0], nil 152 } 153 154 //Download returns reader for downloaded storage object 155 func (s *memoryStorageService) Download(object Object) (io.ReadCloser, error) { 156 var urlPath, err = s.getPath(object.URL()) 157 if err != nil { 158 return nil, err 159 } 160 var pathFragments = strings.Split(urlPath, "/") 161 node, err := s.getFolder(pathFragments) 162 if err != nil { 163 return nil, err 164 } 165 var pathLeaf = pathFragments[len(pathFragments)-1] 166 if memoryFile, ok := node.files[pathLeaf]; ok { 167 return ioutil.NopCloser(bytes.NewReader(memoryFile.content)), nil 168 } 169 return nil, noSuchFileOrDirectoryError 170 } 171 172 //Upload uploads provided reader content for supplied url. 173 func (s *memoryStorageService) Upload(URL string, reader io.Reader) error { 174 return s.UploadWithMode(URL, DefaultFileMode, reader) 175 } 176 177 //Upload uploads provided reader content for supplied url. 178 func (s *memoryStorageService) UploadWithMode(URL string, mode os.FileMode, reader io.Reader) error { 179 urlPath, err := s.getPath(URL) 180 if err != nil { 181 return err 182 } 183 content, err := ioutil.ReadAll(reader) 184 if err != nil { 185 return err 186 } 187 188 var node = s.root 189 var pathFragments = strings.Split(urlPath, "/") 190 for i := 1; i+1 < len(pathFragments); i++ { 191 pathFragment := pathFragments[i] 192 if subFolder, ok := node.folders[pathFragment]; ok { 193 node = subFolder 194 } else { 195 var folderURL = MemoryProviderScheme + "://" + strings.Join(pathFragments[:i+1], "/") 196 var folderInfo = NewFileInfo(pathFragment, 102, folderMode, time.Now(), true) 197 newFolder := newMemoryFolder(folderURL, folderInfo) 198 node.mutext.Lock() 199 node.folders[folderInfo.Name()] = newFolder 200 node.mutext.Unlock() 201 node = newFolder 202 } 203 } 204 205 var pathLeaf = pathFragments[len(pathFragments)-1] 206 fileInfo := NewFileInfo(pathLeaf, int64(len(content)), fileMode, time.Now(), false) 207 var memoryFile = &MemoryFile{name: URL, content: content, fileInfo: fileInfo} 208 node.files[fileInfo.Name()] = memoryFile 209 return nil 210 } 211 212 func (s *memoryStorageService) Register(schema string, service Service) error { 213 return errors.New("unsupported") 214 } 215 216 //Delete removes passed in storage object 217 func (s *memoryStorageService) Delete(object Object) error { 218 var urlPath, err = s.getPath(object.URL()) 219 if err != nil { 220 return err 221 } 222 var pathFragments = strings.Split(urlPath, "/") 223 node, err := s.getFolder(pathFragments) 224 if err != nil { 225 return err 226 } 227 var pathLeaf = pathFragments[len(pathFragments)-1] 228 if _, ok := node.files[pathLeaf]; ok { 229 delete(node.files, pathLeaf) 230 return nil 231 } 232 if _, ok := node.files[pathLeaf]; ok { 233 delete(node.folders, pathLeaf) 234 return nil 235 } 236 return noSuchFileOrDirectoryError 237 } 238 239 //DownloadWithURL downloads content for passed in object URL 240 func (s *memoryStorageService) DownloadWithURL(URL string) (io.ReadCloser, error) { 241 object, err := s.StorageObject(URL) 242 if err != nil { 243 return nil, err 244 } 245 return s.Download(object) 246 } 247 248 // creates a new private memory service 249 func NewPrivateMemoryService() Service { 250 return &memoryStorageService{ 251 root: newMemoryFolder("mem:///", NewFileInfo("/", 102, folderMode, time.Now(), true)), 252 } 253 } 254 255 //creates a new memory service 256 func NewMemoryService() Service { 257 return &memoryStorageService{ 258 root: MemoryRoot, 259 } 260 } 261 262 func init() { 263 Registry().Registry[MemoryProviderScheme] = memServiceProvider 264 } 265 266 func memServiceProvider(credentialFile string) (Service, error) { 267 return NewMemoryService(), nil 268 }