github.com/openimsdk/tools@v0.0.49/s3/kodo/kodo.go (about)

     1  // Copyright © 2023 OpenIM. All rights reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //	http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  package kodo
    15  
    16  import (
    17  	"context"
    18  	"crypto/hmac"
    19  	"crypto/sha1"
    20  	"encoding/base64"
    21  	"encoding/json"
    22  	"errors"
    23  	"fmt"
    24  	"io"
    25  	"net/http"
    26  	"net/url"
    27  	"strconv"
    28  	"strings"
    29  	"time"
    30  
    31  	"github.com/aws/aws-sdk-go-v2/aws"
    32  	awss3config "github.com/aws/aws-sdk-go-v2/config"
    33  	"github.com/aws/aws-sdk-go-v2/credentials"
    34  	awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
    35  	awss3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
    36  	"github.com/openimsdk/tools/errs"
    37  	"github.com/openimsdk/tools/s3"
    38  	"github.com/qiniu/go-sdk/v7/auth"
    39  	"github.com/qiniu/go-sdk/v7/storage"
    40  )
    41  
    42  const (
    43  	minPartSize = 1024 * 1024 * 1        // 1MB
    44  	maxPartSize = 1024 * 1024 * 1024 * 5 // 5GB
    45  	maxNumSize  = 10000
    46  )
    47  
    48  const successCode = http.StatusOK
    49  
    50  type Config struct {
    51  	Endpoint        string
    52  	Bucket          string
    53  	BucketURL       string
    54  	AccessKeyID     string
    55  	AccessKeySecret string
    56  	SessionToken    string
    57  	PublicRead      bool
    58  }
    59  
    60  type Kodo struct {
    61  	AccessKey     string
    62  	SecretKey     string
    63  	Region        string
    64  	Token         string
    65  	Endpoint      string
    66  	BucketURL     string
    67  	Auth          *auth.Credentials
    68  	Client        *awss3.Client
    69  	PresignClient *awss3.PresignClient
    70  }
    71  
    72  func NewKodo(conf Config) (*Kodo, error) {
    73  	//init client
    74  	cfg, err := awss3config.LoadDefaultConfig(context.TODO(),
    75  		awss3config.WithRegion(conf.Bucket),
    76  		awss3config.WithEndpointResolverWithOptions(
    77  			aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
    78  				return aws.Endpoint{URL: conf.Endpoint}, nil
    79  			})),
    80  		awss3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
    81  			conf.AccessKeyID,
    82  			conf.AccessKeySecret,
    83  			conf.SessionToken),
    84  		),
    85  	)
    86  	if err != nil {
    87  		panic(err)
    88  	}
    89  	client := awss3.NewFromConfig(cfg)
    90  	presignClient := awss3.NewPresignClient(client)
    91  
    92  	return &Kodo{
    93  		AccessKey:     conf.AccessKeyID,
    94  		SecretKey:     conf.AccessKeySecret,
    95  		Region:        conf.Bucket,
    96  		BucketURL:     conf.BucketURL,
    97  		Auth:          auth.New(conf.AccessKeyID, conf.AccessKeySecret),
    98  		Client:        client,
    99  		PresignClient: presignClient,
   100  	}, nil
   101  }
   102  
   103  func (k Kodo) Engine() string {
   104  	return "kodo"
   105  }
   106  
   107  func (k Kodo) PartLimit() *s3.PartLimit {
   108  	return &s3.PartLimit{
   109  		MinPartSize: minPartSize,
   110  		MaxPartSize: maxPartSize,
   111  		MaxNumSize:  maxNumSize,
   112  	}
   113  }
   114  
   115  func (k Kodo) InitiateMultipartUpload(ctx context.Context, name string) (*s3.InitiateMultipartUploadResult, error) {
   116  	result, err := k.Client.CreateMultipartUpload(ctx, &awss3.CreateMultipartUploadInput{
   117  		Bucket: aws.String(k.Region),
   118  		Key:    aws.String(name),
   119  	})
   120  	if err != nil {
   121  		return nil, err
   122  	}
   123  	return &s3.InitiateMultipartUploadResult{
   124  		UploadID: aws.ToString(result.UploadId),
   125  		Bucket:   aws.ToString(result.Bucket),
   126  		Key:      aws.ToString(result.Key),
   127  	}, nil
   128  }
   129  
   130  func (k Kodo) CompleteMultipartUpload(ctx context.Context, uploadID string, name string, parts []s3.Part) (*s3.CompleteMultipartUploadResult, error) {
   131  	kodoParts := make([]awss3types.CompletedPart, len(parts))
   132  	for i, part := range parts {
   133  		kodoParts[i] = awss3types.CompletedPart{
   134  			PartNumber: aws.Int32(int32(part.PartNumber)),
   135  			ETag:       aws.String(part.ETag),
   136  		}
   137  	}
   138  	result, err := k.Client.CompleteMultipartUpload(ctx, &awss3.CompleteMultipartUploadInput{
   139  		Bucket:          aws.String(k.Region),
   140  		Key:             aws.String(name),
   141  		UploadId:        aws.String(uploadID),
   142  		MultipartUpload: &awss3types.CompletedMultipartUpload{Parts: kodoParts},
   143  	})
   144  	if err != nil {
   145  		return nil, err
   146  	}
   147  	return &s3.CompleteMultipartUploadResult{
   148  		Location: aws.ToString(result.Location),
   149  		Bucket:   aws.ToString(result.Bucket),
   150  		Key:      aws.ToString(result.Key),
   151  		ETag:     strings.ToLower(strings.ReplaceAll(aws.ToString(result.ETag), `"`, ``)),
   152  	}, nil
   153  }
   154  
   155  func (k Kodo) PartSize(ctx context.Context, size int64) (int64, error) {
   156  	if size <= 0 {
   157  		return 0, errors.New("size must be greater than 0")
   158  	}
   159  	if size > int64(maxPartSize)*int64(maxNumSize) {
   160  		return 0, fmt.Errorf("size must be less than %db", int64(maxPartSize)*int64(maxNumSize))
   161  	}
   162  	if size <= int64(maxPartSize)*int64(maxNumSize) {
   163  		return minPartSize, nil
   164  	}
   165  	partSize := size / maxNumSize
   166  	if size%maxNumSize != 0 {
   167  		partSize++
   168  	}
   169  	return partSize, nil
   170  }
   171  
   172  func (k Kodo) AuthSign(ctx context.Context, uploadID string, name string, expire time.Duration, partNumbers []int) (*s3.AuthSignResult, error) {
   173  	result := s3.AuthSignResult{
   174  		URL:    k.BucketURL + "/" + name,
   175  		Query:  url.Values{"uploadId": {uploadID}},
   176  		Header: make(http.Header),
   177  		Parts:  make([]s3.SignPart, len(partNumbers)),
   178  	}
   179  	for i, partNumber := range partNumbers {
   180  		part, _ := k.PresignClient.PresignUploadPart(ctx, &awss3.UploadPartInput{
   181  			Bucket:     aws.String(k.Region),
   182  			UploadId:   aws.String(uploadID),
   183  			Key:        aws.String(name),
   184  			PartNumber: aws.Int32(int32(partNumber)),
   185  		})
   186  		result.Parts[i] = s3.SignPart{
   187  			PartNumber: partNumber,
   188  			URL:        part.URL,
   189  			Header:     part.SignedHeader,
   190  		}
   191  	}
   192  	return &result, nil
   193  
   194  }
   195  
   196  func (k Kodo) PresignedPutObject(ctx context.Context, name string, expire time.Duration) (string, error) {
   197  	object, err := k.PresignClient.PresignPutObject(ctx, &awss3.PutObjectInput{
   198  		Bucket: aws.String(k.Region),
   199  		Key:    aws.String(name),
   200  	}, func(po *awss3.PresignOptions) {
   201  		po.Expires = expire
   202  	})
   203  	return object.URL, err
   204  
   205  }
   206  
   207  func (k Kodo) DeleteObject(ctx context.Context, name string) error {
   208  	_, err := k.Client.DeleteObject(ctx, &awss3.DeleteObjectInput{
   209  		Bucket: aws.String(k.Region),
   210  		Key:    aws.String(name),
   211  	})
   212  	return err
   213  }
   214  
   215  func (k Kodo) CopyObject(ctx context.Context, src string, dst string) (*s3.CopyObjectInfo, error) {
   216  	result, err := k.Client.CopyObject(ctx, &awss3.CopyObjectInput{
   217  		Bucket:     aws.String(k.Region),
   218  		CopySource: aws.String(k.Region + "/" + src),
   219  		Key:        aws.String(dst),
   220  	})
   221  	if err != nil {
   222  		return nil, err
   223  	}
   224  	return &s3.CopyObjectInfo{
   225  		Key:  dst,
   226  		ETag: strings.ToLower(strings.ReplaceAll(aws.ToString(result.CopyObjectResult.ETag), `"`, ``)),
   227  	}, nil
   228  }
   229  
   230  func (k Kodo) StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) {
   231  	info, err := k.Client.HeadObject(ctx, &awss3.HeadObjectInput{
   232  		Bucket: aws.String(k.Region),
   233  		Key:    aws.String(name),
   234  	})
   235  	if err != nil {
   236  		return nil, err
   237  	}
   238  	res := &s3.ObjectInfo{Key: name}
   239  	res.Size = aws.ToInt64(info.ContentLength)
   240  	res.ETag = strings.ToLower(strings.ReplaceAll(aws.ToString(info.ETag), `"`, ``))
   241  	return res, nil
   242  }
   243  
   244  func (k Kodo) IsNotFound(err error) bool {
   245  	if err != nil {
   246  		var errorType *awss3types.NotFound
   247  		if errors.As(err, &errorType) {
   248  			return true
   249  		}
   250  	}
   251  	return false
   252  }
   253  
   254  func (k Kodo) AbortMultipartUpload(ctx context.Context, uploadID string, name string) error {
   255  	_, err := k.Client.AbortMultipartUpload(ctx, &awss3.AbortMultipartUploadInput{
   256  		UploadId: aws.String(uploadID),
   257  		Bucket:   aws.String(k.Region),
   258  		Key:      aws.String(name),
   259  	})
   260  	return err
   261  }
   262  
   263  func (k Kodo) ListUploadedParts(ctx context.Context, uploadID string, name string, partNumberMarker int, maxParts int) (*s3.ListUploadedPartsResult, error) {
   264  	result, err := k.Client.ListParts(ctx, &awss3.ListPartsInput{
   265  		Key:              aws.String(name),
   266  		UploadId:         aws.String(uploadID),
   267  		Bucket:           aws.String(k.Region),
   268  		MaxParts:         aws.Int32(int32(maxParts)),
   269  		PartNumberMarker: aws.String(strconv.Itoa(partNumberMarker)),
   270  	})
   271  	if err != nil {
   272  		return nil, err
   273  	}
   274  	res := &s3.ListUploadedPartsResult{
   275  		Key:           aws.ToString(result.Key),
   276  		UploadID:      aws.ToString(result.UploadId),
   277  		MaxParts:      int(aws.ToInt32(result.MaxParts)),
   278  		UploadedParts: make([]s3.UploadedPart, len(result.Parts)),
   279  	}
   280  	// int to string
   281  	NextPartNumberMarker, err := strconv.Atoi(aws.ToString(result.NextPartNumberMarker))
   282  	if err != nil {
   283  		return nil, err
   284  	}
   285  	res.NextPartNumberMarker = NextPartNumberMarker
   286  	for i, part := range result.Parts {
   287  		res.UploadedParts[i] = s3.UploadedPart{
   288  			PartNumber:   int(aws.ToInt32(part.PartNumber)),
   289  			LastModified: aws.ToTime(part.LastModified),
   290  			ETag:         aws.ToString(part.ETag),
   291  			Size:         aws.ToInt64(part.Size),
   292  		}
   293  	}
   294  	return res, nil
   295  }
   296  
   297  func (k Kodo) AccessURL(ctx context.Context, name string, expire time.Duration, opt *s3.AccessURLOption) (string, error) {
   298  	//get object head
   299  	info, err := k.Client.HeadObject(ctx, &awss3.HeadObjectInput{
   300  		Bucket: aws.String(k.Region),
   301  		Key:    aws.String(name),
   302  	})
   303  	if err != nil {
   304  		return "", errors.New("AccessURL object not found")
   305  	}
   306  	if opt != nil {
   307  		if opt.ContentType != aws.ToString(info.ContentType) {
   308  			err := k.SetObjectContentType(ctx, name, opt.ContentType)
   309  			if err != nil {
   310  				return "", errors.New("AccessURL setContentType error")
   311  			}
   312  		}
   313  	}
   314  	imageMogr := ""
   315  	//image dispose
   316  	if opt != nil {
   317  		if opt.Image != nil {
   318  			//https://developer.qiniu.com/dora/8255/the-zoom
   319  			process := ""
   320  			if opt.Image.Width > 0 {
   321  				process += strconv.Itoa(opt.Image.Width) + "x"
   322  			}
   323  			if opt.Image.Height > 0 {
   324  				if opt.Image.Width > 0 {
   325  					process += strconv.Itoa(opt.Image.Height)
   326  				} else {
   327  					process += "x" + strconv.Itoa(opt.Image.Height)
   328  				}
   329  			}
   330  			imageMogr = "imageMogr2/thumbnail/" + process
   331  		}
   332  	}
   333  	//expire
   334  	deadline := time.Now().Add(time.Second * expire).Unix()
   335  	domain := k.BucketURL
   336  	query := url.Values{}
   337  	if opt != nil && opt.Filename != "" {
   338  		query.Add("attname", opt.Filename)
   339  	}
   340  	privateURL := storage.MakePrivateURLv2WithQuery(k.Auth, domain, name, query, deadline)
   341  	if imageMogr != "" {
   342  		privateURL += "&" + imageMogr
   343  	}
   344  	return privateURL, nil
   345  }
   346  
   347  func (k *Kodo) SetObjectContentType(ctx context.Context, name string, contentType string) error {
   348  	//set object content-type
   349  	_, err := k.Client.CopyObject(ctx, &awss3.CopyObjectInput{
   350  		Bucket:            aws.String(k.Region),
   351  		CopySource:        aws.String(k.Region + "/" + name),
   352  		Key:               aws.String(name),
   353  		ContentType:       aws.String(contentType),
   354  		MetadataDirective: awss3types.MetadataDirectiveReplace,
   355  	})
   356  	return err
   357  }
   358  func (k *Kodo) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) {
   359  	// https://developer.qiniu.com/kodo/1312/upload
   360  	now := time.Now()
   361  	expiration := now.Add(duration)
   362  	resourceKey := k.Region + ":" + name
   363  	putPolicy := map[string]any{
   364  		"scope":    resourceKey,
   365  		"deadline": now.Unix() + 3600,
   366  	}
   367  
   368  	putPolicyJson, err := json.Marshal(putPolicy)
   369  	if err != nil {
   370  		return nil, errs.WrapMsg(err, "Marshal json error")
   371  	}
   372  	encodedPutPolicy := base64.StdEncoding.EncodeToString(putPolicyJson)
   373  	sign := encodedPutPolicy
   374  	h := hmac.New(sha1.New, []byte(k.SecretKey))
   375  	if _, err := io.WriteString(h, sign); err != nil {
   376  		return nil, errs.WrapMsg(err, "WriteString error")
   377  	}
   378  
   379  	encodedSign := base64.StdEncoding.EncodeToString([]byte(sign))
   380  	uploadToken := k.AccessKey + ":" + encodedSign + ":" + encodedPutPolicy
   381  
   382  	fd := &s3.FormData{
   383  		URL:     k.BucketURL,
   384  		File:    "file",
   385  		Expires: expiration,
   386  		FormData: map[string]string{
   387  			"key":   resourceKey,
   388  			"token": uploadToken,
   389  		},
   390  		SuccessCodes: []int{successCode},
   391  	}
   392  	if contentType != "" {
   393  		fd.FormData["accept"] = contentType
   394  	}
   395  	return fd, nil
   396  }