github.com/viant/toolbox@v0.34.5/storage/mem_service.go (about)

     1  package storage
     2  
     3  import (
     4  	"bytes"
     5  	"errors"
     6  	"io"
     7  	"io/ioutil"
     8  	"net/url"
     9  	"os"
    10  	"strings"
    11  	"sync"
    12  	"time"
    13  )
    14  
    15  const MemoryProviderScheme = "mem"
    16  
    17  var noSuchFileOrDirectoryError = errors.New("No such file or directory")
    18  var folderMode, _ = NewFileMode("drwxrwxrwx")
    19  
    20  //MemoryRoot represents memory root storage
    21  var MemoryRoot = newMemoryFolder("mem:///", NewFileInfo("/", 102, folderMode, time.Now(), true))
    22  
    23  //ResetMemory reset memory root storage
    24  func ResetMemory() {
    25  	MemoryRoot = newMemoryFolder("mem:///", NewFileInfo("/", 102, folderMode, time.Now(), true))
    26  }
    27  
    28  //Service represents memory storage service intended for testing
    29  type memoryStorageService struct {
    30  	root *MemoryFolder
    31  }
    32  
    33  type MemoryFile struct {
    34  	name     string
    35  	fileInfo os.FileInfo
    36  	content  []byte
    37  }
    38  
    39  func (f *MemoryFile) Object() Object {
    40  	return NewAbstractStorageObject(f.name, f, f.fileInfo)
    41  }
    42  
    43  type MemoryFolder struct {
    44  	name     string
    45  	fileInfo os.FileInfo
    46  	mutext   *sync.RWMutex
    47  	files    map[string]*MemoryFile
    48  	folders  map[string]*MemoryFolder
    49  }
    50  
    51  func (f *MemoryFolder) Object() Object {
    52  	return NewAbstractStorageObject(f.name, f, f.fileInfo)
    53  }
    54  
    55  func (f *MemoryFolder) Objects() []Object {
    56  	var result = make([]Object, 0)
    57  	result = append(result, f.Object())
    58  	for _, folder := range f.folders {
    59  		result = append(result, folder.Object())
    60  	}
    61  	for _, file := range f.files {
    62  		result = append(result, file.Object())
    63  	}
    64  	return result
    65  }
    66  
    67  func newMemoryFolder(name string, info os.FileInfo) *MemoryFolder {
    68  	return &MemoryFolder{
    69  		name:     name,
    70  		fileInfo: info,
    71  		mutext:   &sync.RWMutex{},
    72  		files:    make(map[string]*MemoryFile),
    73  		folders:  make(map[string]*MemoryFolder),
    74  	}
    75  }
    76  
    77  func (s *memoryStorageService) getFolder(pathFragments []string) (*MemoryFolder, error) {
    78  	node := s.root
    79  	var ok bool
    80  	for i := 1; i+1 < len(pathFragments); i++ {
    81  		pathFragment := pathFragments[i]
    82  		node, ok = node.folders[pathFragment]
    83  		if !ok {
    84  			return nil, noSuchFileOrDirectoryError
    85  		}
    86  	}
    87  	return node, nil
    88  }
    89  
    90  func (s *memoryStorageService) getPath(URL string) (string, error) {
    91  	parsedURL, err := url.Parse(URL)
    92  	if err != nil {
    93  		return "", err
    94  	}
    95  	parsedPath := parsedURL.Path
    96  	strings.Replace(parsedPath, "//", "/", len(parsedPath))
    97  	if len(parsedPath) > 1 && strings.HasSuffix(parsedPath, "/") {
    98  		parsedPath = string(parsedPath[:len(parsedPath)-1])
    99  	}
   100  	return parsedPath, nil
   101  }
   102  
   103  //List returns a list of object for supplied url
   104  func (s *memoryStorageService) List(URL string) ([]Object, error) {
   105  	path, err := s.getPath(URL)
   106  	if err != nil {
   107  		return nil, err
   108  	}
   109  	if path == "/" {
   110  		return s.root.Objects(), nil
   111  	}
   112  	var pathFragments = strings.Split(path, "/")
   113  	node, err := s.getFolder(pathFragments)
   114  	if err != nil {
   115  		return nil, err
   116  	}
   117  	var pathLeaf = pathFragments[len(pathFragments)-1]
   118  
   119  	if memoryFile, ok := node.files[pathLeaf]; ok {
   120  		return []Object{memoryFile.Object()}, nil
   121  	}
   122  	if folder, ok := node.folders[pathLeaf]; ok {
   123  		return folder.Objects(), nil
   124  	}
   125  
   126  	return []Object{}, nil
   127  }
   128  
   129  //Exists returns true if resource exists
   130  func (s *memoryStorageService) Exists(URL string) (bool, error) {
   131  	objects, err := s.List(URL)
   132  	if err != nil {
   133  		return false, err
   134  	}
   135  	return len(objects) > 0, nil
   136  }
   137  
   138  func (s *memoryStorageService) Close() error {
   139  	return nil
   140  }
   141  
   142  //Object returns a Object for supplied url
   143  func (s *memoryStorageService) StorageObject(URL string) (Object, error) {
   144  	objects, err := s.List(URL)
   145  	if err != nil {
   146  		return nil, err
   147  	}
   148  	if len(objects) == 0 {
   149  		return nil, noSuchFileOrDirectoryError
   150  	}
   151  	return objects[0], nil
   152  }
   153  
   154  //Download returns reader for downloaded storage object
   155  func (s *memoryStorageService) Download(object Object) (io.ReadCloser, error) {
   156  	var urlPath, err = s.getPath(object.URL())
   157  	if err != nil {
   158  		return nil, err
   159  	}
   160  	var pathFragments = strings.Split(urlPath, "/")
   161  	node, err := s.getFolder(pathFragments)
   162  	if err != nil {
   163  		return nil, err
   164  	}
   165  	var pathLeaf = pathFragments[len(pathFragments)-1]
   166  	if memoryFile, ok := node.files[pathLeaf]; ok {
   167  		return ioutil.NopCloser(bytes.NewReader(memoryFile.content)), nil
   168  	}
   169  	return nil, noSuchFileOrDirectoryError
   170  }
   171  
   172  //Upload uploads provided reader content for supplied url.
   173  func (s *memoryStorageService) Upload(URL string, reader io.Reader) error {
   174  	return s.UploadWithMode(URL, DefaultFileMode, reader)
   175  }
   176  
   177  //Upload uploads provided reader content for supplied url.
   178  func (s *memoryStorageService) UploadWithMode(URL string, mode os.FileMode, reader io.Reader) error {
   179  	urlPath, err := s.getPath(URL)
   180  	if err != nil {
   181  		return err
   182  	}
   183  	content, err := ioutil.ReadAll(reader)
   184  	if err != nil {
   185  		return err
   186  	}
   187  
   188  	var node = s.root
   189  	var pathFragments = strings.Split(urlPath, "/")
   190  	for i := 1; i+1 < len(pathFragments); i++ {
   191  		pathFragment := pathFragments[i]
   192  		if subFolder, ok := node.folders[pathFragment]; ok {
   193  			node = subFolder
   194  		} else {
   195  			var folderURL = MemoryProviderScheme + "://" + strings.Join(pathFragments[:i+1], "/")
   196  			var folderInfo = NewFileInfo(pathFragment, 102, folderMode, time.Now(), true)
   197  			newFolder := newMemoryFolder(folderURL, folderInfo)
   198  			node.mutext.Lock()
   199  			node.folders[folderInfo.Name()] = newFolder
   200  			node.mutext.Unlock()
   201  			node = newFolder
   202  		}
   203  	}
   204  
   205  	var pathLeaf = pathFragments[len(pathFragments)-1]
   206  	fileInfo := NewFileInfo(pathLeaf, int64(len(content)), fileMode, time.Now(), false)
   207  	var memoryFile = &MemoryFile{name: URL, content: content, fileInfo: fileInfo}
   208  	node.files[fileInfo.Name()] = memoryFile
   209  	return nil
   210  }
   211  
   212  func (s *memoryStorageService) Register(schema string, service Service) error {
   213  	return errors.New("unsupported")
   214  }
   215  
   216  //Delete removes passed in storage object
   217  func (s *memoryStorageService) Delete(object Object) error {
   218  	var urlPath, err = s.getPath(object.URL())
   219  	if err != nil {
   220  		return err
   221  	}
   222  	var pathFragments = strings.Split(urlPath, "/")
   223  	node, err := s.getFolder(pathFragments)
   224  	if err != nil {
   225  		return err
   226  	}
   227  	var pathLeaf = pathFragments[len(pathFragments)-1]
   228  	if _, ok := node.files[pathLeaf]; ok {
   229  		delete(node.files, pathLeaf)
   230  		return nil
   231  	}
   232  	if _, ok := node.files[pathLeaf]; ok {
   233  		delete(node.folders, pathLeaf)
   234  		return nil
   235  	}
   236  	return noSuchFileOrDirectoryError
   237  }
   238  
   239  //DownloadWithURL downloads content for passed in object URL
   240  func (s *memoryStorageService) DownloadWithURL(URL string) (io.ReadCloser, error) {
   241  	object, err := s.StorageObject(URL)
   242  	if err != nil {
   243  		return nil, err
   244  	}
   245  	return s.Download(object)
   246  }
   247  
   248  // creates a new private memory service
   249  func NewPrivateMemoryService() Service {
   250  	return &memoryStorageService{
   251  		root: newMemoryFolder("mem:///", NewFileInfo("/", 102, folderMode, time.Now(), true)),
   252  	}
   253  }
   254  
   255  //creates a new memory service
   256  func NewMemoryService() Service {
   257  	return &memoryStorageService{
   258  		root: MemoryRoot,
   259  	}
   260  }
   261  
   262  func init() {
   263  	Registry().Registry[MemoryProviderScheme] = memServiceProvider
   264  }
   265  
   266  func memServiceProvider(credentialFile string) (Service, error) {
   267  	return NewMemoryService(), nil
   268  }