github.com/cloudreve/Cloudreve/v3@v3.0.0-20240224133659-3edb00a6484c/pkg/filesystem/driver/oss/handler.go (about) 1 package oss 2 3 import ( 4 "context" 5 "encoding/base64" 6 "encoding/json" 7 "errors" 8 "fmt" 9 "io" 10 "net/url" 11 "path" 12 "path/filepath" 13 "strings" 14 "time" 15 16 "github.com/HFO4/aliyun-oss-go-sdk/oss" 17 model "github.com/cloudreve/Cloudreve/v3/models" 18 "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk" 19 "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff" 20 "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver" 21 "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" 22 "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response" 23 "github.com/cloudreve/Cloudreve/v3/pkg/request" 24 "github.com/cloudreve/Cloudreve/v3/pkg/serializer" 25 "github.com/cloudreve/Cloudreve/v3/pkg/util" 26 ) 27 28 // UploadPolicy 阿里云OSS上传策略 29 type UploadPolicy struct { 30 Expiration string `json:"expiration"` 31 Conditions []interface{} `json:"conditions"` 32 } 33 34 // CallbackPolicy 回调策略 35 type CallbackPolicy struct { 36 CallbackURL string `json:"callbackUrl"` 37 CallbackBody string `json:"callbackBody"` 38 CallbackBodyType string `json:"callbackBodyType"` 39 } 40 41 // Driver 阿里云OSS策略适配器 42 type Driver struct { 43 Policy *model.Policy 44 client *oss.Client 45 bucket *oss.Bucket 46 HTTPClient request.Client 47 } 48 49 type key int 50 51 const ( 52 chunkRetrySleep = time.Duration(5) * time.Second 53 54 // MultiPartUploadThreshold 服务端使用分片上传的阈值 55 MultiPartUploadThreshold uint64 = 5 * (1 << 30) // 5GB 56 // VersionID 文件版本标识 57 VersionID key = iota 58 ) 59 60 func NewDriver(policy *model.Policy) (*Driver, error) { 61 if policy.OptionsSerialized.ChunkSize == 0 { 62 policy.OptionsSerialized.ChunkSize = 25 << 20 // 25 MB 63 } 64 65 driver := &Driver{ 66 Policy: policy, 67 HTTPClient: request.NewClient(), 68 } 69 70 return driver, driver.InitOSSClient(false) 71 } 72 73 // CORS 创建跨域策略 74 func (handler *Driver) CORS() error { 75 return handler.client.SetBucketCORS(handler.Policy.BucketName, []oss.CORSRule{ 76 { 77 AllowedOrigin: []string{"*"}, 78 AllowedMethod: []string{ 79 "GET", 80 "POST", 81 "PUT", 82 "DELETE", 83 "HEAD", 84 }, 85 ExposeHeader: []string{}, 86 AllowedHeader: []string{"*"}, 87 MaxAgeSeconds: 3600, 88 }, 89 }) 90 } 91 92 // InitOSSClient 初始化OSS鉴权客户端 93 func (handler *Driver) InitOSSClient(forceUsePublicEndpoint bool) error { 94 if handler.Policy == nil { 95 return errors.New("empty policy") 96 } 97 98 // 决定是否使用内网 Endpoint 99 endpoint := handler.Policy.Server 100 if handler.Policy.OptionsSerialized.ServerSideEndpoint != "" && !forceUsePublicEndpoint { 101 endpoint = handler.Policy.OptionsSerialized.ServerSideEndpoint 102 } 103 104 // 初始化客户端 105 client, err := oss.New(endpoint, handler.Policy.AccessKey, handler.Policy.SecretKey) 106 if err != nil { 107 return err 108 } 109 handler.client = client 110 111 // 初始化存储桶 112 bucket, err := client.Bucket(handler.Policy.BucketName) 113 if err != nil { 114 return err 115 } 116 handler.bucket = bucket 117 118 return nil 119 } 120 121 // List 列出OSS上的文件 122 func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) { 123 // 列取文件 124 base = strings.TrimPrefix(base, "/") 125 if base != "" { 126 base += "/" 127 } 128 129 var ( 130 delimiter string 131 marker string 132 objects []oss.ObjectProperties 133 commons []string 134 ) 135 if !recursive { 136 delimiter = "/" 137 } 138 139 for { 140 subRes, err := handler.bucket.ListObjects(oss.Marker(marker), oss.Prefix(base), 141 oss.MaxKeys(1000), oss.Delimiter(delimiter)) 142 if err != nil { 143 return nil, err 144 } 145 objects = append(objects, subRes.Objects...) 146 commons = append(commons, subRes.CommonPrefixes...) 147 marker = subRes.NextMarker 148 if marker == "" { 149 break 150 } 151 } 152 153 // 处理列取结果 154 res := make([]response.Object, 0, len(objects)+len(commons)) 155 // 处理目录 156 for _, object := range commons { 157 rel, err := filepath.Rel(base, object) 158 if err != nil { 159 continue 160 } 161 res = append(res, response.Object{ 162 Name: path.Base(object), 163 RelativePath: filepath.ToSlash(rel), 164 Size: 0, 165 IsDir: true, 166 LastModify: time.Now(), 167 }) 168 } 169 // 处理文件 170 for _, object := range objects { 171 rel, err := filepath.Rel(base, object.Key) 172 if err != nil { 173 continue 174 } 175 res = append(res, response.Object{ 176 Name: path.Base(object.Key), 177 Source: object.Key, 178 RelativePath: filepath.ToSlash(rel), 179 Size: uint64(object.Size), 180 IsDir: false, 181 LastModify: object.LastModified, 182 }) 183 } 184 185 return res, nil 186 } 187 188 // Get 获取文件 189 func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) { 190 // 通过VersionID禁止缓存 191 ctx = context.WithValue(ctx, VersionID, time.Now().UnixNano()) 192 193 // 尽可能使用私有 Endpoint 194 ctx = context.WithValue(ctx, fsctx.ForceUsePublicEndpointCtx, false) 195 196 // 获取文件源地址 197 downloadURL, err := handler.Source(ctx, path, int64(model.GetIntSetting("preview_timeout", 60)), false, 0) 198 if err != nil { 199 return nil, err 200 } 201 202 // 获取文件数据流 203 resp, err := handler.HTTPClient.Request( 204 "GET", 205 downloadURL, 206 nil, 207 request.WithContext(ctx), 208 request.WithTimeout(time.Duration(0)), 209 ).CheckHTTPResponse(200).GetRSCloser() 210 if err != nil { 211 return nil, err 212 } 213 214 resp.SetFirstFakeChunk() 215 216 // 尝试自主获取文件大小 217 if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok { 218 resp.SetContentLength(int64(file.Size)) 219 } 220 221 return resp, nil 222 } 223 224 // Put 将文件流保存到指定目录 225 func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error { 226 defer file.Close() 227 fileInfo := file.Info() 228 229 // 凭证有效期 230 credentialTTL := model.GetIntSetting("upload_session_timeout", 3600) 231 232 // 是否允许覆盖 233 overwrite := fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite 234 options := []oss.Option{ 235 oss.Expires(time.Now().Add(time.Duration(credentialTTL) * time.Second)), 236 oss.ForbidOverWrite(!overwrite), 237 } 238 239 // 小文件直接上传 240 if fileInfo.Size < MultiPartUploadThreshold { 241 return handler.bucket.PutObject(fileInfo.SavePath, file, options...) 242 } 243 244 // 超过阈值时使用分片上传 245 imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...) 246 if err != nil { 247 return fmt.Errorf("failed to initiate multipart upload: %w", err) 248 } 249 250 chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{ 251 Max: model.GetIntSetting("chunk_retries", 5), 252 Sleep: chunkRetrySleep, 253 }, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer"))) 254 255 uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error { 256 _, err := handler.bucket.UploadPart(imur, content, current.Length(), current.Index()+1) 257 return err 258 } 259 260 for chunks.Next() { 261 if err := chunks.Process(uploadFunc); err != nil { 262 return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err) 263 } 264 } 265 266 _, err = handler.bucket.CompleteMultipartUpload(imur, oss.CompleteAll("yes"), oss.ForbidOverWrite(!overwrite)) 267 return err 268 } 269 270 // Delete 删除一个或多个文件, 271 // 返回未删除的文件 272 func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) { 273 // 删除文件 274 delRes, err := handler.bucket.DeleteObjects(files) 275 276 if err != nil { 277 return files, err 278 } 279 280 // 统计未删除的文件 281 failed := util.SliceDifference(files, delRes.DeletedObjects) 282 if len(failed) > 0 { 283 return failed, errors.New("failed to delete") 284 } 285 286 return []string{}, nil 287 } 288 289 // Thumb 获取文件缩略图 290 func (handler *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) { 291 // quick check by extension name 292 // https://help.aliyun.com/document_detail/183902.html 293 supported := []string{"png", "jpg", "jpeg", "gif", "bmp", "webp", "heic", "tiff", "avif"} 294 if len(handler.Policy.OptionsSerialized.ThumbExts) > 0 { 295 supported = handler.Policy.OptionsSerialized.ThumbExts 296 } 297 298 if !util.IsInExtensionList(supported, file.Name) || file.Size > (20<<(10*2)) { 299 return nil, driver.ErrorThumbNotSupported 300 } 301 302 // 初始化客户端 303 if err := handler.InitOSSClient(true); err != nil { 304 return nil, err 305 } 306 307 var ( 308 thumbSize = [2]uint{400, 300} 309 ok = false 310 ) 311 if thumbSize, ok = ctx.Value(fsctx.ThumbSizeCtx).([2]uint); !ok { 312 return nil, errors.New("failed to get thumbnail size") 313 } 314 315 thumbEncodeQuality := model.GetIntSetting("thumb_encode_quality", 85) 316 317 thumbParam := fmt.Sprintf("image/resize,m_lfit,h_%d,w_%d/quality,q_%d", thumbSize[1], thumbSize[0], thumbEncodeQuality) 318 ctx = context.WithValue(ctx, fsctx.ThumbSizeCtx, thumbParam) 319 thumbOption := []oss.Option{oss.Process(thumbParam)} 320 thumbURL, err := handler.signSourceURL( 321 ctx, 322 file.SourceName, 323 int64(model.GetIntSetting("preview_timeout", 60)), 324 thumbOption, 325 ) 326 if err != nil { 327 return nil, err 328 } 329 330 return &response.ContentResponse{ 331 Redirect: true, 332 URL: thumbURL, 333 }, nil 334 } 335 336 // Source 获取外链URL 337 func (handler *Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) { 338 // 初始化客户端 339 usePublicEndpoint := true 340 if forceUsePublicEndpoint, ok := ctx.Value(fsctx.ForceUsePublicEndpointCtx).(bool); ok { 341 usePublicEndpoint = forceUsePublicEndpoint 342 } 343 if err := handler.InitOSSClient(usePublicEndpoint); err != nil { 344 return "", err 345 } 346 347 // 尝试从上下文获取文件名 348 fileName := "" 349 if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok { 350 fileName = file.Name 351 } 352 353 // 添加各项设置 354 var signOptions = make([]oss.Option, 0, 2) 355 if isDownload { 356 signOptions = append(signOptions, oss.ResponseContentDisposition("attachment; filename=\""+url.PathEscape(fileName)+"\"")) 357 } 358 if speed > 0 { 359 // Byte 转换为 bit 360 speed *= 8 361 362 // OSS对速度值有范围限制 363 if speed < 819200 { 364 speed = 819200 365 } 366 if speed > 838860800 { 367 speed = 838860800 368 } 369 signOptions = append(signOptions, oss.TrafficLimitParam(int64(speed))) 370 } 371 372 return handler.signSourceURL(ctx, path, ttl, signOptions) 373 } 374 375 func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64, options []oss.Option) (string, error) { 376 signedURL, err := handler.bucket.SignURL(path, oss.HTTPGet, ttl, options...) 377 if err != nil { 378 return "", err 379 } 380 381 // 将最终生成的签名URL域名换成用户自定义的加速域名(如果有) 382 finalURL, err := url.Parse(signedURL) 383 if err != nil { 384 return "", err 385 } 386 387 // 公有空间替换掉Key及不支持的头 388 if !handler.Policy.IsPrivate { 389 query := finalURL.Query() 390 query.Del("OSSAccessKeyId") 391 query.Del("Signature") 392 query.Del("response-content-disposition") 393 query.Del("x-oss-traffic-limit") 394 finalURL.RawQuery = query.Encode() 395 } 396 397 if handler.Policy.BaseURL != "" { 398 cdnURL, err := url.Parse(handler.Policy.BaseURL) 399 if err != nil { 400 return "", err 401 } 402 finalURL.Host = cdnURL.Host 403 finalURL.Scheme = cdnURL.Scheme 404 } 405 406 return finalURL.String(), nil 407 } 408 409 // Token 获取上传策略和认证Token 410 func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) { 411 // 初始化客户端 412 if err := handler.InitOSSClient(true); err != nil { 413 return nil, err 414 } 415 416 // 生成回调地址 417 siteURL := model.GetSiteURL() 418 apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + uploadSession.Key) 419 apiURL := siteURL.ResolveReference(apiBaseURI) 420 421 // 回调策略 422 callbackPolicy := CallbackPolicy{ 423 CallbackURL: apiURL.String(), 424 CallbackBody: `{"name":${x:fname},"source_name":${object},"size":${size},"pic_info":"${imageInfo.width},${imageInfo.height}"}`, 425 CallbackBodyType: "application/json", 426 } 427 callbackPolicyJSON, err := json.Marshal(callbackPolicy) 428 if err != nil { 429 return nil, fmt.Errorf("failed to encode callback policy: %w", err) 430 } 431 callbackPolicyEncoded := base64.StdEncoding.EncodeToString(callbackPolicyJSON) 432 433 // 初始化分片上传 434 fileInfo := file.Info() 435 options := []oss.Option{ 436 oss.Expires(time.Now().Add(time.Duration(ttl) * time.Second)), 437 oss.ForbidOverWrite(true), 438 oss.ContentType(fileInfo.DetectMimeType()), 439 } 440 imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...) 441 if err != nil { 442 return nil, fmt.Errorf("failed to initialize multipart upload: %w", err) 443 } 444 uploadSession.UploadID = imur.UploadID 445 446 // 为每个分片签名上传 URL 447 chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}, false) 448 urls := make([]string, chunks.Num()) 449 for chunks.Next() { 450 err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error { 451 signedURL, err := handler.bucket.SignURL(fileInfo.SavePath, oss.HTTPPut, ttl, 452 oss.PartNumber(c.Index()+1), 453 oss.UploadID(imur.UploadID), 454 oss.ContentType("application/octet-stream")) 455 if err != nil { 456 return err 457 } 458 459 urls[c.Index()] = signedURL 460 return nil 461 }) 462 if err != nil { 463 return nil, err 464 } 465 } 466 467 // 签名完成分片上传的URL 468 completeURL, err := handler.bucket.SignURL(fileInfo.SavePath, oss.HTTPPost, ttl, 469 oss.ContentType("application/octet-stream"), 470 oss.UploadID(imur.UploadID), 471 oss.Expires(time.Now().Add(time.Duration(ttl)*time.Second)), 472 oss.CompleteAll("yes"), 473 oss.ForbidOverWrite(true), 474 oss.CallbackParam(callbackPolicyEncoded)) 475 if err != nil { 476 return nil, err 477 } 478 479 return &serializer.UploadCredential{ 480 SessionID: uploadSession.Key, 481 ChunkSize: handler.Policy.OptionsSerialized.ChunkSize, 482 UploadID: imur.UploadID, 483 UploadURLs: urls, 484 CompleteURL: completeURL, 485 }, nil 486 } 487 488 // 取消上传凭证 489 func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error { 490 return handler.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{UploadID: uploadSession.UploadID, Key: uploadSession.SavePath}, nil) 491 }