github.com/viant/toolbox@v0.34.5/storage/mapper.go (about)

     1  package storage
     2  
     3  import (
     4  	"fmt"
     5  	"github.com/viant/toolbox"
     6  	"io"
     7  	"io/ioutil"
     8  	"os"
     9  	"path"
    10  	"strings"
    11  )
    12  
    13  type StorageMapping struct {
    14  	SourceURL        string
    15  	SourceCredential string
    16  	DestinationURI   string
    17  	TargetFile       string
    18  	TargetPackage    string
    19  	UseTextFormat    bool
    20  }
    21  
    22  var binaryFormats = map[string]bool{
    23  	".png":  true,
    24  	".jpeg": true,
    25  	".ico":  true,
    26  	".jpg":  true,
    27  }
    28  
    29  //GenerateStorageCode create a *.go files with statically scanned content from source URL.
    30  func GenerateStorageCode(mappings ...*StorageMapping) error {
    31  	destinationService := NewMemoryService()
    32  	for _, mapping := range mappings {
    33  		sourceService, err := NewServiceForURL(mapping.SourceURL, mapping.SourceCredential)
    34  		if err != nil {
    35  			return err
    36  		}
    37  		handler, writer, err := NewStorageMapperHandler(mapping.TargetFile, mapping.TargetPackage, mapping.UseTextFormat, binaryFormats)
    38  		if err != nil {
    39  			return err
    40  		}
    41  		defer writer.Close()
    42  		destinationURL := "mem://" + mapping.DestinationURI
    43  		err = copyStorageContent(sourceService, mapping.SourceURL, destinationService, destinationURL, nil, "", handler)
    44  		if err != nil {
    45  			return err
    46  		}
    47  	}
    48  	return nil
    49  }
    50  
    51  //NewStorageMapperHandler creates a template handler for generating go file that write static content into memory service.
    52  func NewStorageMapperHandler(filename, pkg string, useTextFormat bool, binaryFormat map[string]bool) (CopyHandler, io.WriteCloser, error) {
    53  	_ = toolbox.RemoveFileIfExist(filename)
    54  	writer, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, 0644)
    55  	if err != nil {
    56  		return nil, nil, err
    57  	}
    58  	template := &templateWriter{writer}
    59  	_ = template.Init(pkg)
    60  	return func(sourceObject Object, source io.Reader, destinationService Service, destinationURL string) error {
    61  
    62  		_, name := toolbox.URLSplit(destinationURL)
    63  		if strings.HasPrefix(name, ".") {
    64  			//skip hidden files
    65  			return nil
    66  		}
    67  		content, err := ioutil.ReadAll(source)
    68  		if err != nil {
    69  			return err
    70  		}
    71  		isText := useTextFormat
    72  		ext := path.Ext(destinationURL)
    73  		if binaryFormat[ext] {
    74  			isText = false
    75  		}
    76  
    77  		template.WriteStorageContent(destinationURL, content, isText)
    78  		return nil
    79  	}, template, nil
    80  }
    81  
    82  type templateWriter struct {
    83  	io.WriteCloser
    84  }
    85  
    86  func (t *templateWriter) Init(pkg string) error {
    87  	var begin = `package %v
    88  
    89  import (
    90  	"bytes"
    91  	"github.com/viant/toolbox/storage"
    92  	"log"
    93  )
    94  
    95  func init() {
    96  	var memStorage = storage.NewMemoryService();
    97  `
    98  	_, err := t.Write([]byte(fmt.Sprintf(begin, pkg)))
    99  	return err
   100  }
   101  
   102  func (t *templateWriter) WriteStorageContent(URL string, content []byte, useText bool) error {
   103  	if useText {
   104  		text := string(content)
   105  		var count = strings.Count(text, "`")
   106  		if count > 0 {
   107  			text = strings.Replace(text, "`", "`+\"`\"+`", count)
   108  			content = []byte(text)
   109  		}
   110  	}
   111  	var contentReader = fmt.Sprintf("bytes.NewReader([]byte(`%s`))", content)
   112  	if !toolbox.IsASCIIText(contentReader) && !useText {
   113  		var byteArray = make([]string, 0)
   114  		for _, b := range content {
   115  			byteArray = append(byteArray, fmt.Sprintf("%d", b))
   116  		}
   117  		contentReader = fmt.Sprintf("bytes.NewReader([]byte{%v})", strings.Join(byteArray, ","))
   118  	}
   119  	var payload = `	{
   120  		err := memStorage.Upload("%v", %v)
   121  		if err != nil {
   122  			log.Printf("failed to upload: %v %v", err)
   123  		}
   124  	}
   125  `
   126  	payload = fmt.Sprintf(payload, URL, contentReader, URL, "%v")
   127  	_, err := t.Write([]byte(payload))
   128  	return err
   129  }
   130  
   131  func (t *templateWriter) Close() error {
   132  	var end = "}\n"
   133  	_, err := t.Write([]byte(end))
   134  	t.WriteCloser.Close()
   135  	return err
   136  }