github.com/cloudreve/Cloudreve/v3@v3.0.0-20240224133659-3edb00a6484c/pkg/filesystem/driver/remote/client.go (about) 1 package remote 2 3 import ( 4 "context" 5 "encoding/json" 6 "fmt" 7 model "github.com/cloudreve/Cloudreve/v3/models" 8 "github.com/cloudreve/Cloudreve/v3/pkg/auth" 9 "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk" 10 "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff" 11 "github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx" 12 "github.com/cloudreve/Cloudreve/v3/pkg/request" 13 "github.com/cloudreve/Cloudreve/v3/pkg/serializer" 14 "github.com/cloudreve/Cloudreve/v3/pkg/util" 15 "github.com/gofrs/uuid" 16 "io" 17 "net/http" 18 "net/url" 19 "path" 20 "strings" 21 "time" 22 ) 23 24 const ( 25 basePath = "/api/v3/slave/" 26 OverwriteHeader = auth.CrHeaderPrefix + "Overwrite" 27 chunkRetrySleep = time.Duration(5) * time.Second 28 ) 29 30 // Client to operate uploading to remote slave server 31 type Client interface { 32 // CreateUploadSession creates remote upload session 33 CreateUploadSession(ctx context.Context, session *serializer.UploadSession, ttl int64, overwrite bool) error 34 // GetUploadURL signs an url for uploading file 35 GetUploadURL(ttl int64, sessionID string) (string, string, error) 36 // Upload uploads file to remote server 37 Upload(ctx context.Context, file fsctx.FileHeader) error 38 // DeleteUploadSession deletes remote upload session 39 DeleteUploadSession(ctx context.Context, sessionID string) error 40 } 41 42 // NewClient creates new Client from given policy 43 func NewClient(policy *model.Policy) (Client, error) { 44 authInstance := auth.HMACAuth{[]byte(policy.SecretKey)} 45 serverURL, err := url.Parse(policy.Server) 46 if err != nil { 47 return nil, err 48 } 49 50 base, _ := url.Parse(basePath) 51 signTTL := model.GetIntSetting("slave_api_timeout", 60) 52 53 return &remoteClient{ 54 policy: policy, 55 authInstance: authInstance, 56 httpClient: request.NewClient( 57 request.WithEndpoint(serverURL.ResolveReference(base).String()), 58 request.WithCredential(authInstance, int64(signTTL)), 59 request.WithMasterMeta(), 60 request.WithSlaveMeta(policy.AccessKey), 61 ), 62 }, nil 63 } 64 65 type remoteClient struct { 66 policy *model.Policy 67 authInstance auth.Auth 68 httpClient request.Client 69 } 70 71 func (c *remoteClient) Upload(ctx context.Context, file fsctx.FileHeader) error { 72 ttl := model.GetIntSetting("upload_session_timeout", 86400) 73 fileInfo := file.Info() 74 session := &serializer.UploadSession{ 75 Key: uuid.Must(uuid.NewV4()).String(), 76 VirtualPath: fileInfo.VirtualPath, 77 Name: fileInfo.FileName, 78 Size: fileInfo.Size, 79 SavePath: fileInfo.SavePath, 80 LastModified: fileInfo.LastModified, 81 Policy: *c.policy, 82 } 83 84 // Create upload session 85 overwrite := fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite 86 if err := c.CreateUploadSession(ctx, session, int64(ttl), overwrite); err != nil { 87 return fmt.Errorf("failed to create upload session: %w", err) 88 } 89 90 // Initial chunk groups 91 chunks := chunk.NewChunkGroup(file, c.policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{ 92 Max: model.GetIntSetting("chunk_retries", 5), 93 Sleep: chunkRetrySleep, 94 }, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer"))) 95 96 uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error { 97 return c.uploadChunk(ctx, session.Key, current.Index(), content, overwrite, current.Length()) 98 } 99 100 // upload chunks 101 for chunks.Next() { 102 if err := chunks.Process(uploadFunc); err != nil { 103 if err := c.DeleteUploadSession(ctx, session.Key); err != nil { 104 util.Log().Warning("failed to delete upload session: %s", err) 105 } 106 107 return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err) 108 } 109 } 110 111 return nil 112 } 113 114 func (c *remoteClient) DeleteUploadSession(ctx context.Context, sessionID string) error { 115 resp, err := c.httpClient.Request( 116 "DELETE", 117 "upload/"+sessionID, 118 nil, 119 request.WithContext(ctx), 120 ).CheckHTTPResponse(200).DecodeResponse() 121 if err != nil { 122 return err 123 } 124 125 if resp.Code != 0 { 126 return serializer.NewErrorFromResponse(resp) 127 } 128 129 return nil 130 } 131 132 func (c *remoteClient) CreateUploadSession(ctx context.Context, session *serializer.UploadSession, ttl int64, overwrite bool) error { 133 reqBodyEncoded, err := json.Marshal(map[string]interface{}{ 134 "session": session, 135 "ttl": ttl, 136 "overwrite": overwrite, 137 }) 138 if err != nil { 139 return err 140 } 141 142 bodyReader := strings.NewReader(string(reqBodyEncoded)) 143 resp, err := c.httpClient.Request( 144 "PUT", 145 "upload", 146 bodyReader, 147 request.WithContext(ctx), 148 ).CheckHTTPResponse(200).DecodeResponse() 149 if err != nil { 150 return err 151 } 152 153 if resp.Code != 0 { 154 return serializer.NewErrorFromResponse(resp) 155 } 156 157 return nil 158 } 159 160 func (c *remoteClient) GetUploadURL(ttl int64, sessionID string) (string, string, error) { 161 base, err := url.Parse(c.policy.Server) 162 if err != nil { 163 return "", "", err 164 } 165 166 base.Path = path.Join(base.Path, basePath, "upload", sessionID) 167 req, err := http.NewRequest("POST", base.String(), nil) 168 if err != nil { 169 return "", "", err 170 } 171 172 req = auth.SignRequest(c.authInstance, req, ttl) 173 return req.URL.String(), req.Header["Authorization"][0], nil 174 } 175 176 func (c *remoteClient) uploadChunk(ctx context.Context, sessionID string, index int, chunk io.Reader, overwrite bool, size int64) error { 177 resp, err := c.httpClient.Request( 178 "POST", 179 fmt.Sprintf("upload/%s?chunk=%d", sessionID, index), 180 chunk, 181 request.WithContext(ctx), 182 request.WithTimeout(time.Duration(0)), 183 request.WithContentLength(size), 184 request.WithHeader(map[string][]string{OverwriteHeader: {fmt.Sprintf("%t", overwrite)}}), 185 ).CheckHTTPResponse(200).DecodeResponse() 186 if err != nil { 187 return err 188 } 189 190 if resp.Code != 0 { 191 return serializer.NewErrorFromResponse(resp) 192 } 193 194 return nil 195 }