github.com/machinefi/w3bstream@v1.6.5-rc9.0.20240426031326-b8c7c4876e72/pkg/depends/conf/filesystem/s3/s3.go (about) 1 package confs3 2 3 import ( 4 "bytes" 5 "context" 6 "io" 7 "net/url" 8 "time" 9 10 "github.com/aws/aws-sdk-go/aws/awserr" 11 "github.com/minio/minio-go/v7" 12 "github.com/minio/minio-go/v7/pkg/credentials" 13 14 "github.com/machinefi/w3bstream/pkg/depends/base/types" 15 "github.com/machinefi/w3bstream/pkg/depends/conf/filesystem" 16 ) 17 18 type S3Endpoint interface { 19 Endpoint() string 20 AccessKeyID() string 21 SecretAccessKey() string 22 BucketName() string 23 Secure() bool 24 } 25 26 type PresignedFn func(db *ObjectDB, key string, exp time.Duration) url.Values 27 28 // ObjectDB Deprecated 29 type ObjectDB struct { 30 Endpoint string 31 Region string 32 AccessKeyID string 33 SecretAccessKey types.Password 34 BucketName string 35 Secure bool 36 UrlExpire types.Duration 37 Presigned PresignedFn `env:"-"` 38 } 39 40 func (db *ObjectDB) SetDefault() { 41 if db.UrlExpire == 0 { 42 db.UrlExpire = types.Duration(10 * time.Minute) 43 } 44 } 45 46 func (db *ObjectDB) LivenessCheck() map[string]string { 47 key := db.BucketName + "." + db.Endpoint 48 m := map[string]string{ 49 key: "ok", 50 } 51 52 c, err := db.Client() 53 54 if err != nil { 55 m[key] = err.Error() 56 } else { 57 if _, err := c.GetBucketLocation(context.Background(), db.BucketName); err != nil { 58 m[key] = err.Error() 59 } 60 } 61 62 return m 63 } 64 65 func (db *ObjectDB) Client() (*minio.Client, error) { 66 options := &minio.Options{ 67 Creds: credentials.NewStaticV4(db.AccessKeyID, db.SecretAccessKey.String(), ""), 68 Secure: db.Secure, 69 Region: db.Region, 70 } 71 72 client, err := minio.New(db.Endpoint, options) 73 if err != nil { 74 return nil, err 75 } 76 return client, nil 77 } 78 79 func (db *ObjectDB) PublicURL(meta *filesystem.ObjectMeta) *url.URL { 80 u := &url.URL{} 81 u.Scheme = "http" 82 if db.Secure { 83 u.Scheme += "s" 84 } 85 86 u.Host = db.Endpoint 87 u.Path = db.BucketName + "/" + meta.Key() 88 return u 89 } 90 91 func (db *ObjectDB) ProtectURL(ctx context.Context, meta *filesystem.ObjectMeta, exp time.Duration) (*url.URL, error) { 92 c, err := db.Client() 93 if err != nil { 94 return nil, err 95 } 96 values := url.Values{} 97 if db.Presigned != nil { 98 values = db.Presigned(db, meta.Key(), exp) 99 } 100 101 u, err := c.PresignedGetObject(ctx, db.BucketName, meta.Key(), exp, values) 102 if err != nil { 103 return nil, err 104 } 105 return u, nil 106 } 107 108 func (db *ObjectDB) PutObject(ctx context.Context, r io.Reader, meta *filesystem.ObjectMeta) error { 109 if ctx == nil { 110 ctx = context.Background() 111 } 112 113 c, err := db.Client() 114 if err != nil { 115 return err 116 } 117 118 if meta.Size == 0 { 119 if canLen, ok := r.(interface{ Len() int }); ok { 120 meta.Size = int64(canLen.Len()) 121 } 122 } 123 124 _, err = c.PutObject( 125 ctx, db.BucketName, meta.Key(), 126 r, meta.Size, minio.PutObjectOptions{ContentType: meta.ContentType}, 127 ) 128 129 return err 130 } 131 132 func (db *ObjectDB) ReadObject(ctx context.Context, w io.Writer, meta *filesystem.ObjectMeta) error { 133 c, err := db.Client() 134 if err != nil { 135 return err 136 } 137 138 object, err := c.GetObject(ctx, db.BucketName, meta.Key(), DefaultGetObjectOptions) 139 if err != nil { 140 return err 141 } 142 defer object.Close() 143 144 _, err = io.Copy(w, object) 145 if err != nil { 146 return err 147 } 148 149 return err 150 } 151 152 func (db *ObjectDB) PresignedPutObject(ctx context.Context, meta *filesystem.ObjectMeta, exp time.Duration) (string, error) { 153 c, err := db.Client() 154 if err != nil { 155 return "", err 156 } 157 address, err := c.PresignedPutObject( 158 ctx, 159 db.BucketName, 160 meta.Key(), 161 exp, 162 ) 163 if err != nil { 164 return "", err 165 } 166 return address.String(), nil 167 } 168 169 func (db *ObjectDB) DeleteObject(ctx context.Context, meta *filesystem.ObjectMeta) error { 170 c, err := db.Client() 171 if err != nil { 172 return err 173 } 174 175 return c.RemoveObject(ctx, db.BucketName, meta.Key(), DefaultRemoveObjectOptions) 176 } 177 178 func (db *ObjectDB) StatsObject(ctx context.Context, meta *filesystem.ObjectMeta) (*filesystem.ObjectMeta, error) { 179 c, err := db.Client() 180 if err != nil { 181 return nil, err 182 } 183 184 info, err := c.StatObject(ctx, db.BucketName, meta.Key(), minio.StatObjectOptions{}) 185 if err != nil { 186 awsErr, ok := err.(awserr.RequestFailure) 187 if ok && awsErr.StatusCode() == 404 { 188 return nil, filesystem.ErrNotExistObjectKey 189 } 190 return nil, err 191 } 192 193 om, err := filesystem.ParseObjectMetaFromKey(info.Key) 194 if err != nil { 195 return nil, err 196 } 197 198 om.ContentType = info.ContentType 199 om.ETag = info.ETag 200 om.Size = info.Size 201 202 return om, err 203 } 204 205 func (db *ObjectDB) ListObjectByGroup(ctx context.Context, grp string) ([]*filesystem.ObjectMeta, error) { 206 c, err := db.Client() 207 if err != nil { 208 return nil, err 209 } 210 211 metas := make([]*filesystem.ObjectMeta, 0) 212 213 objectsCh := c.ListObjects(ctx, db.BucketName, minio.ListObjectsOptions{ 214 Prefix: grp, 215 Recursive: true, 216 }) 217 218 for obj := range objectsCh { 219 om, err := filesystem.ParseObjectMetaFromKey(obj.Key) 220 if err != nil { 221 continue 222 } 223 224 om.ContentType = obj.ContentType 225 om.ETag = obj.ETag 226 om.Size = obj.Size 227 228 metas = append(metas, om) 229 } 230 231 return metas, nil 232 } 233 234 func (db *ObjectDB) Upload(key string, content []byte) error { 235 meta, err := filesystem.ParseObjectMetaFromKey(key) 236 if err != nil { 237 return err 238 } 239 return db.PutObject(context.Background(), bytes.NewBuffer(content), meta) 240 } 241 242 func (db *ObjectDB) Read(key string) ([]byte, error) { 243 meta, err := filesystem.ParseObjectMetaFromKey(key) 244 if err != nil { 245 return nil, err 246 } 247 buf := bytes.NewBuffer(nil) 248 err = db.ReadObject(context.Background(), buf, meta) 249 if err != nil { 250 return nil, err 251 } 252 return buf.Bytes(), err 253 } 254 255 func (db *ObjectDB) Delete(key string) error { 256 meta, err := filesystem.ParseObjectMetaFromKey(key) 257 if err != nil { 258 return err 259 } 260 return db.DeleteObject(context.Background(), meta) 261 } 262 263 func (db *ObjectDB) DownloadUrl(key string) (string, error) { 264 meta, err := filesystem.ParseObjectMetaFromKey(key) 265 if err != nil { 266 return "", err 267 } 268 u, err := db.ProtectURL(context.Background(), meta, db.UrlExpire.Duration()) 269 if err != nil { 270 return "", err 271 } 272 return u.String(), err 273 } 274 275 // StatObject Deprecated 276 func (db *ObjectDB) StatObject(key string) (*filesystem.ObjectMeta, error) { 277 meta, err := filesystem.ParseObjectMetaFromKey(key) 278 if err != nil { 279 return nil, err 280 } 281 return db.StatsObject(context.Background(), meta) 282 } 283 284 var ( 285 DefaultGetObjectOptions = minio.GetObjectOptions{} 286 DefaultRemoveObjectOptions = minio.RemoveObjectOptions{} 287 )