github.com/cloudreve/Cloudreve/v3@v3.0.0-20240224133659-3edb00a6484c/pkg/filesystem/driver/qiniu/handler.go (about) 1 package qiniu 2 3 import ( 4 "context" 5 "encoding/base64" 6 "errors" 7 "fmt" 8 "net/http" 9 "net/url" 10 "path" 11 "path/filepath" 12 "strings" 13 "time" 14 15 model "github.com/cloudreve/Cloudreve/v3/models" 16 "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver" 17 "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" 18 "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response" 19 "github.com/cloudreve/Cloudreve/v3/pkg/request" 20 "github.com/cloudreve/Cloudreve/v3/pkg/serializer" 21 "github.com/cloudreve/Cloudreve/v3/pkg/util" 22 "github.com/qiniu/go-sdk/v7/auth/qbox" 23 "github.com/qiniu/go-sdk/v7/storage" 24 ) 25 26 // Driver 本地策略适配器 27 type Driver struct { 28 Policy *model.Policy 29 mac *qbox.Mac 30 cfg *storage.Config 31 bucket *storage.BucketManager 32 } 33 34 func NewDriver(policy *model.Policy) *Driver { 35 if policy.OptionsSerialized.ChunkSize == 0 { 36 policy.OptionsSerialized.ChunkSize = 25 << 20 // 25 MB 37 } 38 39 mac := qbox.NewMac(policy.AccessKey, policy.SecretKey) 40 cfg := &storage.Config{UseHTTPS: true} 41 return &Driver{ 42 Policy: policy, 43 mac: mac, 44 cfg: cfg, 45 bucket: storage.NewBucketManager(mac, cfg), 46 } 47 } 48 49 // List 列出给定路径下的文件 50 func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) { 51 base = strings.TrimPrefix(base, "/") 52 if base != "" { 53 base += "/" 54 } 55 56 var ( 57 delimiter string 58 marker string 59 objects []storage.ListItem 60 commons []string 61 ) 62 if !recursive { 63 delimiter = "/" 64 } 65 66 for { 67 entries, folders, nextMarker, hashNext, err := handler.bucket.ListFiles( 68 handler.Policy.BucketName, 69 base, delimiter, marker, 1000) 70 if err != nil { 71 return nil, err 72 } 73 objects = append(objects, entries...) 74 commons = append(commons, folders...) 75 if !hashNext { 76 break 77 } 78 marker = nextMarker 79 } 80 81 // 处理列取结果 82 res := make([]response.Object, 0, len(objects)+len(commons)) 83 // 处理目录 84 for _, object := range commons { 85 rel, err := filepath.Rel(base, object) 86 if err != nil { 87 continue 88 } 89 res = append(res, response.Object{ 90 Name: path.Base(object), 91 RelativePath: filepath.ToSlash(rel), 92 Size: 0, 93 IsDir: true, 94 LastModify: time.Now(), 95 }) 96 } 97 // 处理文件 98 for _, object := range objects { 99 rel, err := filepath.Rel(base, object.Key) 100 if err != nil { 101 continue 102 } 103 res = append(res, response.Object{ 104 Name: path.Base(object.Key), 105 Source: object.Key, 106 RelativePath: filepath.ToSlash(rel), 107 Size: uint64(object.Fsize), 108 IsDir: false, 109 LastModify: time.Unix(object.PutTime/10000000, 0), 110 }) 111 } 112 113 return res, nil 114 } 115 116 // Get 获取文件 117 func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) { 118 // 给文件名加上随机参数以强制拉取 119 path = fmt.Sprintf("%s?v=%d", path, time.Now().UnixNano()) 120 121 // 获取文件源地址 122 downloadURL, err := handler.Source(ctx, path, int64(model.GetIntSetting("preview_timeout", 60)), false, 0) 123 if err != nil { 124 return nil, err 125 } 126 127 // 获取文件数据流 128 client := request.NewClient() 129 resp, err := client.Request( 130 "GET", 131 downloadURL, 132 nil, 133 request.WithContext(ctx), 134 request.WithHeader( 135 http.Header{"Cache-Control": {"no-cache", "no-store", "must-revalidate"}}, 136 ), 137 request.WithTimeout(time.Duration(0)), 138 ).CheckHTTPResponse(200).GetRSCloser() 139 if err != nil { 140 return nil, err 141 } 142 143 resp.SetFirstFakeChunk() 144 145 // 尝试自主获取文件大小 146 if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok { 147 resp.SetContentLength(int64(file.Size)) 148 } 149 150 return resp, nil 151 } 152 153 // Put 将文件流保存到指定目录 154 func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error { 155 defer file.Close() 156 157 // 凭证有效期 158 credentialTTL := model.GetIntSetting("upload_session_timeout", 3600) 159 160 // 生成上传策略 161 fileInfo := file.Info() 162 scope := handler.Policy.BucketName 163 if fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite { 164 scope = fmt.Sprintf("%s:%s", handler.Policy.BucketName, fileInfo.SavePath) 165 } 166 167 putPolicy := storage.PutPolicy{ 168 // 指定为覆盖策略 169 Scope: scope, 170 SaveKey: fileInfo.SavePath, 171 ForceSaveKey: true, 172 FsizeLimit: int64(fileInfo.Size), 173 } 174 // 是否开启了MIMEType限制 175 if handler.Policy.OptionsSerialized.MimeType != "" { 176 putPolicy.MimeLimit = handler.Policy.OptionsSerialized.MimeType 177 } 178 179 // 生成上传凭证 180 token, err := handler.getUploadCredential(ctx, putPolicy, fileInfo, int64(credentialTTL), false) 181 if err != nil { 182 return err 183 } 184 185 // 创建上传表单 186 cfg := storage.Config{} 187 formUploader := storage.NewFormUploader(&cfg) 188 ret := storage.PutRet{} 189 putExtra := storage.PutExtra{ 190 Params: map[string]string{}, 191 } 192 193 // 开始上传 194 err = formUploader.Put(ctx, &ret, token.Credential, fileInfo.SavePath, file, int64(fileInfo.Size), &putExtra) 195 if err != nil { 196 return err 197 } 198 199 return nil 200 } 201 202 // Delete 删除一个或多个文件, 203 // 返回未删除的文件 204 func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) { 205 // TODO 大于一千个文件需要分批发送 206 deleteOps := make([]string, 0, len(files)) 207 for _, key := range files { 208 deleteOps = append(deleteOps, storage.URIDelete(handler.Policy.BucketName, key)) 209 } 210 211 rets, err := handler.bucket.Batch(deleteOps) 212 213 // 处理删除结果 214 if err != nil { 215 failed := make([]string, 0, len(rets)) 216 for k, ret := range rets { 217 if ret.Code != 200 && ret.Code != 612 { 218 failed = append(failed, files[k]) 219 } 220 } 221 return failed, errors.New("删除失败") 222 } 223 224 return []string{}, nil 225 } 226 227 // Thumb 获取文件缩略图 228 func (handler *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) { 229 // quick check by extension name 230 // https://developer.qiniu.com/dora/api/basic-processing-images-imageview2 231 supported := []string{"png", "jpg", "jpeg", "gif", "bmp", "webp", "tiff", "avif", "psd"} 232 if len(handler.Policy.OptionsSerialized.ThumbExts) > 0 { 233 supported = handler.Policy.OptionsSerialized.ThumbExts 234 } 235 236 if !util.IsInExtensionList(supported, file.Name) || file.Size > (20<<(10*2)) { 237 return nil, driver.ErrorThumbNotSupported 238 } 239 240 var ( 241 thumbSize = [2]uint{400, 300} 242 ok = false 243 ) 244 if thumbSize, ok = ctx.Value(fsctx.ThumbSizeCtx).([2]uint); !ok { 245 return nil, errors.New("failed to get thumbnail size") 246 } 247 248 thumbEncodeQuality := model.GetIntSetting("thumb_encode_quality", 85) 249 250 thumb := fmt.Sprintf("%s?imageView2/1/w/%d/h/%d/q/%d", file.SourceName, thumbSize[0], thumbSize[1], thumbEncodeQuality) 251 return &response.ContentResponse{ 252 Redirect: true, 253 URL: handler.signSourceURL( 254 ctx, 255 thumb, 256 int64(model.GetIntSetting("preview_timeout", 60)), 257 ), 258 }, nil 259 } 260 261 // Source 获取外链URL 262 func (handler *Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) { 263 // 尝试从上下文获取文件名 264 fileName := "" 265 if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok { 266 fileName = file.Name 267 } 268 269 // 加入下载相关设置 270 if isDownload { 271 path = path + "?attname=" + url.PathEscape(fileName) 272 } 273 274 // 取得原始文件地址 275 return handler.signSourceURL(ctx, path, ttl), nil 276 } 277 278 func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64) string { 279 var sourceURL string 280 if handler.Policy.IsPrivate { 281 deadline := time.Now().Add(time.Second * time.Duration(ttl)).Unix() 282 sourceURL = storage.MakePrivateURL(handler.mac, handler.Policy.BaseURL, path, deadline) 283 } else { 284 sourceURL = storage.MakePublicURL(handler.Policy.BaseURL, path) 285 } 286 return sourceURL 287 } 288 289 // Token 获取上传策略和认证Token 290 func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) { 291 // 生成回调地址 292 siteURL := model.GetSiteURL() 293 apiBaseURI, _ := url.Parse("/api/v3/callback/qiniu/" + uploadSession.Key) 294 apiURL := siteURL.ResolveReference(apiBaseURI) 295 296 // 创建上传策略 297 fileInfo := file.Info() 298 putPolicy := storage.PutPolicy{ 299 Scope: handler.Policy.BucketName, 300 CallbackURL: apiURL.String(), 301 CallbackBody: `{"size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`, 302 CallbackBodyType: "application/json", 303 SaveKey: fileInfo.SavePath, 304 ForceSaveKey: true, 305 FsizeLimit: int64(handler.Policy.MaxSize), 306 } 307 // 是否开启了MIMEType限制 308 if handler.Policy.OptionsSerialized.MimeType != "" { 309 putPolicy.MimeLimit = handler.Policy.OptionsSerialized.MimeType 310 } 311 312 credential, err := handler.getUploadCredential(ctx, putPolicy, fileInfo, ttl, true) 313 if err != nil { 314 return nil, fmt.Errorf("failed to init parts: %w", err) 315 } 316 317 credential.SessionID = uploadSession.Key 318 credential.ChunkSize = handler.Policy.OptionsSerialized.ChunkSize 319 320 uploadSession.UploadURL = credential.UploadURLs[0] 321 uploadSession.Credential = credential.Credential 322 323 return credential, nil 324 } 325 326 // getUploadCredential 签名上传策略并创建上传会话 327 func (handler *Driver) getUploadCredential(ctx context.Context, policy storage.PutPolicy, file *fsctx.UploadTaskInfo, TTL int64, resume bool) (*serializer.UploadCredential, error) { 328 // 上传凭证 329 policy.Expires = uint64(TTL) 330 upToken := policy.UploadToken(handler.mac) 331 332 // 初始化分片上传 333 resumeUploader := storage.NewResumeUploaderV2(handler.cfg) 334 upHost, err := resumeUploader.UpHost(handler.Policy.AccessKey, handler.Policy.BucketName) 335 if err != nil { 336 return nil, err 337 } 338 339 ret := &storage.InitPartsRet{} 340 if resume { 341 err = resumeUploader.InitParts(ctx, upToken, upHost, handler.Policy.BucketName, file.SavePath, true, ret) 342 } 343 344 return &serializer.UploadCredential{ 345 UploadURLs: []string{upHost + "/buckets/" + handler.Policy.BucketName + "/objects/" + base64.URLEncoding.EncodeToString([]byte(file.SavePath)) + "/uploads/" + ret.UploadID}, 346 Credential: upToken, 347 }, err 348 } 349 350 // 取消上传凭证 351 func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error { 352 resumeUploader := storage.NewResumeUploaderV2(handler.cfg) 353 return resumeUploader.Client.CallWith(ctx, nil, "DELETE", uploadSession.UploadURL, http.Header{"Authorization": {"UpToken " + uploadSession.Credential}}, nil, 0) 354 }