github.com/rudderlabs/rudder-go-kit@v0.30.0/filemanager/filemanager.go (about)

     1  //go:generate mockgen -destination=mock_filemanager/mock_filemanager.go -package mock_filemanager github.com/rudderlabs/rudder-go-kit/filemanager FileManager
     2  package filemanager
     3  
     4  import (
     5  	"context"
     6  	"errors"
     7  	"fmt"
     8  	"os"
     9  	"time"
    10  
    11  	"github.com/rudderlabs/rudder-go-kit/config"
    12  	"github.com/rudderlabs/rudder-go-kit/logger"
    13  )
    14  
    15  const defaultTimeout = 120 * time.Second
    16  
    17  var (
    18  	ErrKeyNotFound            = errors.New("NoSuchKey")
    19  	ErrInvalidServiceProvider = errors.New("service provider not supported")
    20  )
    21  
    22  // Factory is a function that returns a new file manager
    23  type Factory func(settings *Settings) (FileManager, error)
    24  
    25  // UploadedFile contains information about the uploaded file
    26  type UploadedFile struct {
    27  	Location   string
    28  	ObjectName string
    29  }
    30  
    31  // FileInfo contains information about a file
    32  type FileInfo struct {
    33  	Key          string
    34  	LastModified time.Time
    35  }
    36  
    37  // FileManager is able to manage files in a storage provider
    38  type FileManager interface {
    39  	// ListFilesWithPrefix starts a list session for files with given prefix
    40  	ListFilesWithPrefix(ctx context.Context, startAfter, prefix string, maxItems int64) ListSession
    41  	// Download downloads the file with given key to the passed in file
    42  	Download(context.Context, *os.File, string) error
    43  	// Upload uploads the passed in file to the file manager
    44  	Upload(context.Context, *os.File, ...string) (UploadedFile, error)
    45  	// Delete deletes the file(s) with given key(s)
    46  	Delete(ctx context.Context, keys []string) error
    47  
    48  	// Prefix returns the prefix for the file manager
    49  	Prefix() string
    50  	// SetTimeout overrides the default timeout for the file manager
    51  	SetTimeout(timeout time.Duration)
    52  
    53  	// GetObjectNameFromLocation gets the object name/key name from the object location url
    54  	GetObjectNameFromLocation(string) (string, error)
    55  	// GetDownloadKeyFromFileLocation gets the download key from the object location url
    56  	GetDownloadKeyFromFileLocation(string) string
    57  }
    58  
    59  // ListSession is a session for listing files
    60  type ListSession interface {
    61  	// Next returns the next batch of files, until there are no more files for this session
    62  	Next() (fileObjects []*FileInfo, err error)
    63  }
    64  
    65  // Settings for file manager
    66  type Settings struct {
    67  	Provider string
    68  	Config   map[string]interface{}
    69  	Logger   logger.Logger
    70  	Conf     *config.Config
    71  }
    72  
    73  // New returns file manager backed by configured provider
    74  func New(settings *Settings) (FileManager, error) {
    75  	log := settings.Logger
    76  	if log == nil {
    77  		log = logger.NewLogger().Child("filemanager")
    78  	}
    79  	conf := settings.Conf
    80  	if conf == nil {
    81  		conf = config.Default
    82  	}
    83  
    84  	switch settings.Provider {
    85  	case "S3_DATALAKE":
    86  		return NewS3Manager(settings.Config, log, getDefaultTimeout(conf, settings.Provider))
    87  	case "S3":
    88  		return NewS3Manager(settings.Config, log, getDefaultTimeout(conf, settings.Provider))
    89  	case "GCS":
    90  		return NewGCSManager(settings.Config, log, getDefaultTimeout(conf, settings.Provider))
    91  	case "AZURE_BLOB":
    92  		return NewAzureBlobManager(settings.Config, log, getDefaultTimeout(conf, settings.Provider))
    93  	case "MINIO":
    94  		return NewMinioManager(settings.Config, log, getDefaultTimeout(conf, settings.Provider))
    95  	case "DIGITAL_OCEAN_SPACES":
    96  		return NewDigitalOceanManager(settings.Config, log, getDefaultTimeout(conf, settings.Provider))
    97  	}
    98  	return nil, fmt.Errorf("%w: %s", ErrInvalidServiceProvider, settings.Provider)
    99  }
   100  
   101  func getDefaultTimeout(config *config.Config, destType string) func() time.Duration {
   102  	return func() time.Duration {
   103  		key := "timeout"
   104  		defaultValueInTimescaleUnits := int64(120)
   105  		timeScale := time.Second
   106  		if config.IsSet("FileManager." + destType + "." + key) {
   107  			return config.GetDuration("FileManager."+destType+"."+key, defaultValueInTimescaleUnits, timeScale)
   108  		}
   109  		if config.IsSet("FileManager." + key) {
   110  			return config.GetDuration("FileManager."+key, defaultValueInTimescaleUnits, timeScale)
   111  		}
   112  		return func() time.Duration { // legacy keys used in rudder-server
   113  			destOverrideFound := config.IsSet("BatchRouter." + destType + "." + key)
   114  			if destOverrideFound {
   115  				return config.GetDuration("BatchRouter."+destType+"."+key, defaultValueInTimescaleUnits, timeScale)
   116  			} else {
   117  				return config.GetDuration("BatchRouter."+key, defaultValueInTimescaleUnits, timeScale)
   118  			}
   119  		}()
   120  	}
   121  }