github.com/machinefi/w3bstream@v1.6.5-rc9.0.20240426031326-b8c7c4876e72/pkg/depends/conf/filesystem/s3/s3.go (about)

     1  package confs3
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"io"
     7  	"net/url"
     8  	"time"
     9  
    10  	"github.com/aws/aws-sdk-go/aws/awserr"
    11  	"github.com/minio/minio-go/v7"
    12  	"github.com/minio/minio-go/v7/pkg/credentials"
    13  
    14  	"github.com/machinefi/w3bstream/pkg/depends/base/types"
    15  	"github.com/machinefi/w3bstream/pkg/depends/conf/filesystem"
    16  )
    17  
    18  type S3Endpoint interface {
    19  	Endpoint() string
    20  	AccessKeyID() string
    21  	SecretAccessKey() string
    22  	BucketName() string
    23  	Secure() bool
    24  }
    25  
    26  type PresignedFn func(db *ObjectDB, key string, exp time.Duration) url.Values
    27  
    28  // ObjectDB Deprecated
    29  type ObjectDB struct {
    30  	Endpoint        string
    31  	Region          string
    32  	AccessKeyID     string
    33  	SecretAccessKey types.Password
    34  	BucketName      string
    35  	Secure          bool
    36  	UrlExpire       types.Duration
    37  	Presigned       PresignedFn `env:"-"`
    38  }
    39  
    40  func (db *ObjectDB) SetDefault() {
    41  	if db.UrlExpire == 0 {
    42  		db.UrlExpire = types.Duration(10 * time.Minute)
    43  	}
    44  }
    45  
    46  func (db *ObjectDB) LivenessCheck() map[string]string {
    47  	key := db.BucketName + "." + db.Endpoint
    48  	m := map[string]string{
    49  		key: "ok",
    50  	}
    51  
    52  	c, err := db.Client()
    53  
    54  	if err != nil {
    55  		m[key] = err.Error()
    56  	} else {
    57  		if _, err := c.GetBucketLocation(context.Background(), db.BucketName); err != nil {
    58  			m[key] = err.Error()
    59  		}
    60  	}
    61  
    62  	return m
    63  }
    64  
    65  func (db *ObjectDB) Client() (*minio.Client, error) {
    66  	options := &minio.Options{
    67  		Creds:  credentials.NewStaticV4(db.AccessKeyID, db.SecretAccessKey.String(), ""),
    68  		Secure: db.Secure,
    69  		Region: db.Region,
    70  	}
    71  
    72  	client, err := minio.New(db.Endpoint, options)
    73  	if err != nil {
    74  		return nil, err
    75  	}
    76  	return client, nil
    77  }
    78  
    79  func (db *ObjectDB) PublicURL(meta *filesystem.ObjectMeta) *url.URL {
    80  	u := &url.URL{}
    81  	u.Scheme = "http"
    82  	if db.Secure {
    83  		u.Scheme += "s"
    84  	}
    85  
    86  	u.Host = db.Endpoint
    87  	u.Path = db.BucketName + "/" + meta.Key()
    88  	return u
    89  }
    90  
    91  func (db *ObjectDB) ProtectURL(ctx context.Context, meta *filesystem.ObjectMeta, exp time.Duration) (*url.URL, error) {
    92  	c, err := db.Client()
    93  	if err != nil {
    94  		return nil, err
    95  	}
    96  	values := url.Values{}
    97  	if db.Presigned != nil {
    98  		values = db.Presigned(db, meta.Key(), exp)
    99  	}
   100  
   101  	u, err := c.PresignedGetObject(ctx, db.BucketName, meta.Key(), exp, values)
   102  	if err != nil {
   103  		return nil, err
   104  	}
   105  	return u, nil
   106  }
   107  
   108  func (db *ObjectDB) PutObject(ctx context.Context, r io.Reader, meta *filesystem.ObjectMeta) error {
   109  	if ctx == nil {
   110  		ctx = context.Background()
   111  	}
   112  
   113  	c, err := db.Client()
   114  	if err != nil {
   115  		return err
   116  	}
   117  
   118  	if meta.Size == 0 {
   119  		if canLen, ok := r.(interface{ Len() int }); ok {
   120  			meta.Size = int64(canLen.Len())
   121  		}
   122  	}
   123  
   124  	_, err = c.PutObject(
   125  		ctx, db.BucketName, meta.Key(),
   126  		r, meta.Size, minio.PutObjectOptions{ContentType: meta.ContentType},
   127  	)
   128  
   129  	return err
   130  }
   131  
   132  func (db *ObjectDB) ReadObject(ctx context.Context, w io.Writer, meta *filesystem.ObjectMeta) error {
   133  	c, err := db.Client()
   134  	if err != nil {
   135  		return err
   136  	}
   137  
   138  	object, err := c.GetObject(ctx, db.BucketName, meta.Key(), DefaultGetObjectOptions)
   139  	if err != nil {
   140  		return err
   141  	}
   142  	defer object.Close()
   143  
   144  	_, err = io.Copy(w, object)
   145  	if err != nil {
   146  		return err
   147  	}
   148  
   149  	return err
   150  }
   151  
   152  func (db *ObjectDB) PresignedPutObject(ctx context.Context, meta *filesystem.ObjectMeta, exp time.Duration) (string, error) {
   153  	c, err := db.Client()
   154  	if err != nil {
   155  		return "", err
   156  	}
   157  	address, err := c.PresignedPutObject(
   158  		ctx,
   159  		db.BucketName,
   160  		meta.Key(),
   161  		exp,
   162  	)
   163  	if err != nil {
   164  		return "", err
   165  	}
   166  	return address.String(), nil
   167  }
   168  
   169  func (db *ObjectDB) DeleteObject(ctx context.Context, meta *filesystem.ObjectMeta) error {
   170  	c, err := db.Client()
   171  	if err != nil {
   172  		return err
   173  	}
   174  
   175  	return c.RemoveObject(ctx, db.BucketName, meta.Key(), DefaultRemoveObjectOptions)
   176  }
   177  
   178  func (db *ObjectDB) StatsObject(ctx context.Context, meta *filesystem.ObjectMeta) (*filesystem.ObjectMeta, error) {
   179  	c, err := db.Client()
   180  	if err != nil {
   181  		return nil, err
   182  	}
   183  
   184  	info, err := c.StatObject(ctx, db.BucketName, meta.Key(), minio.StatObjectOptions{})
   185  	if err != nil {
   186  		awsErr, ok := err.(awserr.RequestFailure)
   187  		if ok && awsErr.StatusCode() == 404 {
   188  			return nil, filesystem.ErrNotExistObjectKey
   189  		}
   190  		return nil, err
   191  	}
   192  
   193  	om, err := filesystem.ParseObjectMetaFromKey(info.Key)
   194  	if err != nil {
   195  		return nil, err
   196  	}
   197  
   198  	om.ContentType = info.ContentType
   199  	om.ETag = info.ETag
   200  	om.Size = info.Size
   201  
   202  	return om, err
   203  }
   204  
   205  func (db *ObjectDB) ListObjectByGroup(ctx context.Context, grp string) ([]*filesystem.ObjectMeta, error) {
   206  	c, err := db.Client()
   207  	if err != nil {
   208  		return nil, err
   209  	}
   210  
   211  	metas := make([]*filesystem.ObjectMeta, 0)
   212  
   213  	objectsCh := c.ListObjects(ctx, db.BucketName, minio.ListObjectsOptions{
   214  		Prefix:    grp,
   215  		Recursive: true,
   216  	})
   217  
   218  	for obj := range objectsCh {
   219  		om, err := filesystem.ParseObjectMetaFromKey(obj.Key)
   220  		if err != nil {
   221  			continue
   222  		}
   223  
   224  		om.ContentType = obj.ContentType
   225  		om.ETag = obj.ETag
   226  		om.Size = obj.Size
   227  
   228  		metas = append(metas, om)
   229  	}
   230  
   231  	return metas, nil
   232  }
   233  
   234  func (db *ObjectDB) Upload(key string, content []byte) error {
   235  	meta, err := filesystem.ParseObjectMetaFromKey(key)
   236  	if err != nil {
   237  		return err
   238  	}
   239  	return db.PutObject(context.Background(), bytes.NewBuffer(content), meta)
   240  }
   241  
   242  func (db *ObjectDB) Read(key string) ([]byte, error) {
   243  	meta, err := filesystem.ParseObjectMetaFromKey(key)
   244  	if err != nil {
   245  		return nil, err
   246  	}
   247  	buf := bytes.NewBuffer(nil)
   248  	err = db.ReadObject(context.Background(), buf, meta)
   249  	if err != nil {
   250  		return nil, err
   251  	}
   252  	return buf.Bytes(), err
   253  }
   254  
   255  func (db *ObjectDB) Delete(key string) error {
   256  	meta, err := filesystem.ParseObjectMetaFromKey(key)
   257  	if err != nil {
   258  		return err
   259  	}
   260  	return db.DeleteObject(context.Background(), meta)
   261  }
   262  
   263  func (db *ObjectDB) DownloadUrl(key string) (string, error) {
   264  	meta, err := filesystem.ParseObjectMetaFromKey(key)
   265  	if err != nil {
   266  		return "", err
   267  	}
   268  	u, err := db.ProtectURL(context.Background(), meta, db.UrlExpire.Duration())
   269  	if err != nil {
   270  		return "", err
   271  	}
   272  	return u.String(), err
   273  }
   274  
   275  // StatObject Deprecated
   276  func (db *ObjectDB) StatObject(key string) (*filesystem.ObjectMeta, error) {
   277  	meta, err := filesystem.ParseObjectMetaFromKey(key)
   278  	if err != nil {
   279  		return nil, err
   280  	}
   281  	return db.StatsObject(context.Background(), meta)
   282  }
   283  
   284  var (
   285  	DefaultGetObjectOptions    = minio.GetObjectOptions{}
   286  	DefaultRemoveObjectOptions = minio.RemoveObjectOptions{}
   287  )