github.com/openimsdk/tools@v0.0.49/s3/kodo/kodo.go (about) 1 // Copyright © 2023 OpenIM. All rights reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 package kodo 15 16 import ( 17 "context" 18 "crypto/hmac" 19 "crypto/sha1" 20 "encoding/base64" 21 "encoding/json" 22 "errors" 23 "fmt" 24 "io" 25 "net/http" 26 "net/url" 27 "strconv" 28 "strings" 29 "time" 30 31 "github.com/aws/aws-sdk-go-v2/aws" 32 awss3config "github.com/aws/aws-sdk-go-v2/config" 33 "github.com/aws/aws-sdk-go-v2/credentials" 34 awss3 "github.com/aws/aws-sdk-go-v2/service/s3" 35 awss3types "github.com/aws/aws-sdk-go-v2/service/s3/types" 36 "github.com/openimsdk/tools/errs" 37 "github.com/openimsdk/tools/s3" 38 "github.com/qiniu/go-sdk/v7/auth" 39 "github.com/qiniu/go-sdk/v7/storage" 40 ) 41 42 const ( 43 minPartSize = 1024 * 1024 * 1 // 1MB 44 maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB 45 maxNumSize = 10000 46 ) 47 48 const successCode = http.StatusOK 49 50 type Config struct { 51 Endpoint string 52 Bucket string 53 BucketURL string 54 AccessKeyID string 55 AccessKeySecret string 56 SessionToken string 57 PublicRead bool 58 } 59 60 type Kodo struct { 61 AccessKey string 62 SecretKey string 63 Region string 64 Token string 65 Endpoint string 66 BucketURL string 67 Auth *auth.Credentials 68 Client *awss3.Client 69 PresignClient *awss3.PresignClient 70 } 71 72 func NewKodo(conf Config) (*Kodo, error) { 73 //init client 74 cfg, err := awss3config.LoadDefaultConfig(context.TODO(), 75 awss3config.WithRegion(conf.Bucket), 76 awss3config.WithEndpointResolverWithOptions( 77 aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { 78 return aws.Endpoint{URL: conf.Endpoint}, nil 79 })), 80 awss3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( 81 conf.AccessKeyID, 82 conf.AccessKeySecret, 83 conf.SessionToken), 84 ), 85 ) 86 if err != nil { 87 panic(err) 88 } 89 client := awss3.NewFromConfig(cfg) 90 presignClient := awss3.NewPresignClient(client) 91 92 return &Kodo{ 93 AccessKey: conf.AccessKeyID, 94 SecretKey: conf.AccessKeySecret, 95 Region: conf.Bucket, 96 BucketURL: conf.BucketURL, 97 Auth: auth.New(conf.AccessKeyID, conf.AccessKeySecret), 98 Client: client, 99 PresignClient: presignClient, 100 }, nil 101 } 102 103 func (k Kodo) Engine() string { 104 return "kodo" 105 } 106 107 func (k Kodo) PartLimit() *s3.PartLimit { 108 return &s3.PartLimit{ 109 MinPartSize: minPartSize, 110 MaxPartSize: maxPartSize, 111 MaxNumSize: maxNumSize, 112 } 113 } 114 115 func (k Kodo) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) { 116 result, err := k.Client.CreateMultipartUpload(ctx, &awss3.CreateMultipartUploadInput{ 117 Bucket: aws.String(k.Region), 118 Key: aws.String(name), 119 }) 120 if err != nil { 121 return nil, err 122 } 123 return &s3.InitiateMultipartUploadResult{ 124 UploadID: aws.ToString(result.UploadId), 125 Bucket: aws.ToString(result.Bucket), 126 Key: aws.ToString(result.Key), 127 }, nil 128 } 129 130 func (k Kodo) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) { 131 kodoParts := make([]awss3types.CompletedPart, len(parts)) 132 for i, part := range parts { 133 kodoParts[i] = awss3types.CompletedPart{ 134 PartNumber: aws.Int32(int32(part.PartNumber)), 135 ETag: aws.String(part.ETag), 136 } 137 } 138 result, err := k.Client.CompleteMultipartUpload(ctx, &awss3.CompleteMultipartUploadInput{ 139 Bucket: aws.String(k.Region), 140 Key: aws.String(name), 141 UploadId: aws.String(uploadID), 142 MultipartUpload: &awss3types.CompletedMultipartUpload{Parts: kodoParts}, 143 }) 144 if err != nil { 145 return nil, err 146 } 147 return &s3.CompleteMultipartUploadResult{ 148 Location: aws.ToString(result.Location), 149 Bucket: aws.ToString(result.Bucket), 150 Key: aws.ToString(result.Key), 151 ETag: strings.ToLower(strings.ReplaceAll(aws.ToString(result.ETag), `"`, ``)), 152 }, nil 153 } 154 155 func (k Kodo) PartSize(ctx context.Context, size int64) (int64, error) { 156 if size <= 0 { 157 return 0, errors.New("size must be greater than 0") 158 } 159 if size > int64(maxPartSize)*int64(maxNumSize) { 160 return 0, fmt.Errorf("size must be less than %db", int64(maxPartSize)*int64(maxNumSize)) 161 } 162 if size <= int64(maxPartSize)*int64(maxNumSize) { 163 return minPartSize, nil 164 } 165 partSize := size / maxNumSize 166 if size%maxNumSize != 0 { 167 partSize++ 168 } 169 return partSize, nil 170 } 171 172 func (k Kodo) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) { 173 result := s3.AuthSignResult{ 174 URL: k.BucketURL + "/" + name, 175 Query: url.Values{"uploadId": {uploadID}}, 176 Header: make(http.Header), 177 Parts: make([]s3.SignPart, len(partNumbers)), 178 } 179 for i, partNumber := range partNumbers { 180 part, _ := k.PresignClient.PresignUploadPart(ctx, &awss3.UploadPartInput{ 181 Bucket: aws.String(k.Region), 182 UploadId: aws.String(uploadID), 183 Key: aws.String(name), 184 PartNumber: aws.Int32(int32(partNumber)), 185 }) 186 result.Parts[i] = s3.SignPart{ 187 PartNumber: partNumber, 188 URL: part.URL, 189 Header: part.SignedHeader, 190 } 191 } 192 return &result, nil 193 194 } 195 196 func (k Kodo) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) { 197 object, err := k.PresignClient.PresignPutObject(ctx, &awss3.PutObjectInput{ 198 Bucket: aws.String(k.Region), 199 Key: aws.String(name), 200 }, func(po *awss3.PresignOptions) { 201 po.Expires = expire 202 }) 203 return object.URL, err 204 205 } 206 207 func (k Kodo) DeleteObject(ctx context.Context, name string) error { 208 _, err := k.Client.DeleteObject(ctx, &awss3.DeleteObjectInput{ 209 Bucket: aws.String(k.Region), 210 Key: aws.String(name), 211 }) 212 return err 213 } 214 215 func (k Kodo) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) { 216 result, err := k.Client.CopyObject(ctx, &awss3.CopyObjectInput{ 217 Bucket: aws.String(k.Region), 218 CopySource: aws.String(k.Region + "/" + src), 219 Key: aws.String(dst), 220 }) 221 if err != nil { 222 return nil, err 223 } 224 return &s3.CopyObjectInfo{ 225 Key: dst, 226 ETag: strings.ToLower(strings.ReplaceAll(aws.ToString(result.CopyObjectResult.ETag), `"`, ``)), 227 }, nil 228 } 229 230 func (k Kodo) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) { 231 info, err := k.Client.HeadObject(ctx, &awss3.HeadObjectInput{ 232 Bucket: aws.String(k.Region), 233 Key: aws.String(name), 234 }) 235 if err != nil { 236 return nil, err 237 } 238 res := &s3.ObjectInfo{Key: name} 239 res.Size = aws.ToInt64(info.ContentLength) 240 res.ETag = strings.ToLower(strings.ReplaceAll(aws.ToString(info.ETag), `"`, ``)) 241 return res, nil 242 } 243 244 func (k Kodo) IsNotFound(err error) bool { 245 if err != nil { 246 var errorType *awss3types.NotFound 247 if errors.As(err, &errorType) { 248 return true 249 } 250 } 251 return false 252 } 253 254 func (k Kodo) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error { 255 _, err := k.Client.AbortMultipartUpload(ctx, &awss3.AbortMultipartUploadInput{ 256 UploadId: aws.String(uploadID), 257 Bucket: aws.String(k.Region), 258 Key: aws.String(name), 259 }) 260 return err 261 } 262 263 func (k Kodo) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) { 264 result, err := k.Client.ListParts(ctx, &awss3.ListPartsInput{ 265 Key: aws.String(name), 266 UploadId: aws.String(uploadID), 267 Bucket: aws.String(k.Region), 268 MaxParts: aws.Int32(int32(maxParts)), 269 PartNumberMarker: aws.String(strconv.Itoa(partNumberMarker)), 270 }) 271 if err != nil { 272 return nil, err 273 } 274 res := &s3.ListUploadedPartsResult{ 275 Key: aws.ToString(result.Key), 276 UploadID: aws.ToString(result.UploadId), 277 MaxParts: int(aws.ToInt32(result.MaxParts)), 278 UploadedParts: make([]s3.UploadedPart, len(result.Parts)), 279 } 280 // int to string 281 NextPartNumberMarker, err := strconv.Atoi(aws.ToString(result.NextPartNumberMarker)) 282 if err != nil { 283 return nil, err 284 } 285 res.NextPartNumberMarker = NextPartNumberMarker 286 for i, part := range result.Parts { 287 res.UploadedParts[i] = s3.UploadedPart{ 288 PartNumber: int(aws.ToInt32(part.PartNumber)), 289 LastModified: aws.ToTime(part.LastModified), 290 ETag: aws.ToString(part.ETag), 291 Size: aws.ToInt64(part.Size), 292 } 293 } 294 return res, nil 295 } 296 297 func (k Kodo) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) { 298 //get object head 299 info, err := k.Client.HeadObject(ctx, &awss3.HeadObjectInput{ 300 Bucket: aws.String(k.Region), 301 Key: aws.String(name), 302 }) 303 if err != nil { 304 return "", errors.New("AccessURL object not found") 305 } 306 if opt != nil { 307 if opt.ContentType != aws.ToString(info.ContentType) { 308 err := k.SetObjectContentType(ctx, name, opt.ContentType) 309 if err != nil { 310 return "", errors.New("AccessURL setContentType error") 311 } 312 } 313 } 314 imageMogr := "" 315 //image dispose 316 if opt != nil { 317 if opt.Image != nil { 318 //https://developer.qiniu.com/dora/8255/the-zoom 319 process := "" 320 if opt.Image.Width > 0 { 321 process += strconv.Itoa(opt.Image.Width) + "x" 322 } 323 if opt.Image.Height > 0 { 324 if opt.Image.Width > 0 { 325 process += strconv.Itoa(opt.Image.Height) 326 } else { 327 process += "x" + strconv.Itoa(opt.Image.Height) 328 } 329 } 330 imageMogr = "imageMogr2/thumbnail/" + process 331 } 332 } 333 //expire 334 deadline := time.Now().Add(time.Second * expire).Unix() 335 domain := k.BucketURL 336 query := url.Values{} 337 if opt != nil && opt.Filename != "" { 338 query.Add("attname", opt.Filename) 339 } 340 privateURL := storage.MakePrivateURLv2WithQuery(k.Auth, domain, name, query, deadline) 341 if imageMogr != "" { 342 privateURL += "&" + imageMogr 343 } 344 return privateURL, nil 345 } 346 347 func (k *Kodo) SetObjectContentType(ctx context.Context, name string, contentType string) error { 348 //set object content-type 349 _, err := k.Client.CopyObject(ctx, &awss3.CopyObjectInput{ 350 Bucket: aws.String(k.Region), 351 CopySource: aws.String(k.Region + "/" + name), 352 Key: aws.String(name), 353 ContentType: aws.String(contentType), 354 MetadataDirective: awss3types.MetadataDirectiveReplace, 355 }) 356 return err 357 } 358 func (k *Kodo) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { 359 // https://developer.qiniu.com/kodo/1312/upload 360 now := time.Now() 361 expiration := now.Add(duration) 362 resourceKey := k.Region + ":" + name 363 putPolicy := map[string]any{ 364 "scope": resourceKey, 365 "deadline": now.Unix() + 3600, 366 } 367 368 putPolicyJson, err := json.Marshal(putPolicy) 369 if err != nil { 370 return nil, errs.WrapMsg(err, "Marshal json error") 371 } 372 encodedPutPolicy := base64.StdEncoding.EncodeToString(putPolicyJson) 373 sign := encodedPutPolicy 374 h := hmac.New(sha1.New, []byte(k.SecretKey)) 375 if _, err := io.WriteString(h, sign); err != nil { 376 return nil, errs.WrapMsg(err, "WriteString error") 377 } 378 379 encodedSign := base64.StdEncoding.EncodeToString([]byte(sign)) 380 uploadToken := k.AccessKey + ":" + encodedSign + ":" + encodedPutPolicy 381 382 fd := &s3.FormData{ 383 URL: k.BucketURL, 384 File: "file", 385 Expires: expiration, 386 FormData: map[string]string{ 387 "key": resourceKey, 388 "token": uploadToken, 389 }, 390 SuccessCodes: []int{successCode}, 391 } 392 if contentType != "" { 393 fd.FormData["accept"] = contentType 394 } 395 return fd, nil 396 }