github.com/gaukas/goofys100m@v0.24.0/internal/backend_gcs3.go (about) 1 // Copyright 2019 Ka-Hing Cheung 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package internal 16 17 import ( 18 . "github.com/kahing/goofys/api/common" 19 20 "fmt" 21 "io" 22 "net/url" 23 "strconv" 24 "sync" 25 "sync/atomic" 26 27 "github.com/aws/aws-sdk-go/service/s3" 28 29 "github.com/jacobsa/fuse" 30 ) 31 32 // GCS variant of S3 33 type GCS3 struct { 34 *S3Backend 35 } 36 37 type GCSMultipartBlobCommitInput struct { 38 Size uint64 39 ETag *string 40 Prev *MultipartBlobAddInput 41 } 42 43 func NewGCS3(bucket string, flags *FlagStorage, config *S3Config) (*GCS3, error) { 44 s3Backend, err := NewS3(bucket, flags, config) 45 if err != nil { 46 return nil, err 47 } 48 s3Backend.Capabilities().Name = "gcs" 49 s := &GCS3{S3Backend: s3Backend} 50 s.S3Backend.gcs = true 51 s.S3Backend.cap.NoParallelMultipart = true 52 return s, nil 53 } 54 55 func (s *GCS3) Delegate() interface{} { 56 return s 57 } 58 59 func (s *GCS3) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error) { 60 // GCS does not have multi-delete 61 var wg sync.WaitGroup 62 var overallErr error 63 64 for _, key := range param.Items { 65 wg.Add(1) 66 go func(key string) { 67 _, err := s.DeleteBlob(&DeleteBlobInput{ 68 Key: key, 69 }) 70 if err != nil && err != fuse.ENOENT { 71 overallErr = err 72 } 73 wg.Done() 74 }(key) 75 } 76 wg.Wait() 77 if overallErr != nil { 78 return nil, mapAwsError(overallErr) 79 } 80 81 return &DeleteBlobsOutput{}, nil 82 } 83 84 func (s *GCS3) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) { 85 mpu := s3.CreateMultipartUploadInput{ 86 Bucket: &s.bucket, 87 Key: ¶m.Key, 88 StorageClass: &s.config.StorageClass, 89 ContentType: param.ContentType, 90 } 91 92 if s.config.UseSSE { 93 mpu.ServerSideEncryption = &s.sseType 94 if s.config.UseKMS && s.config.KMSKeyID != "" { 95 mpu.SSEKMSKeyId = &s.config.KMSKeyID 96 } 97 } 98 99 if s.config.ACL != "" { 100 mpu.ACL = &s.config.ACL 101 } 102 103 req, _ := s.CreateMultipartUploadRequest(&mpu) 104 // v4 signing of this fails 105 s.setV2Signer(&req.Handlers) 106 // get rid of ?uploads= 107 req.HTTPRequest.URL.RawQuery = "" 108 req.HTTPRequest.Header.Set("x-goog-resumable", "start") 109 110 err := req.Send() 111 if err != nil { 112 s3Log.Errorf("CreateMultipartUpload %v = %v", param.Key, err) 113 return nil, mapAwsError(err) 114 } 115 116 location := req.HTTPResponse.Header.Get("Location") 117 _, err = url.Parse(location) 118 if err != nil { 119 s3Log.Errorf("CreateMultipartUpload %v %v = %v", param.Key, location, err) 120 return nil, mapAwsError(err) 121 } 122 123 return &MultipartBlobCommitInput{ 124 Key: ¶m.Key, 125 Metadata: param.Metadata, 126 UploadId: &location, 127 Parts: make([]*string, 10000), // at most 10K parts 128 backendData: &GCSMultipartBlobCommitInput{}, 129 }, nil 130 } 131 132 func (s *GCS3) uploadPart(param *MultipartBlobAddInput, totalSize uint64, last bool) (etag *string, err error) { 133 atomic.AddUint32(¶m.Commit.NumParts, 1) 134 135 if closer, ok := param.Body.(io.Closer); ok { 136 defer closer.Close() 137 } 138 139 // the mpuId serves as authentication token so 140 // technically we don't need to sign this anymore and 141 // can just use a plain HTTP request, but going 142 // through aws-sdk-go anyway to get retry handling 143 params := &s3.PutObjectInput{ 144 Bucket: &s.bucket, 145 Key: param.Commit.Key, 146 Body: param.Body, 147 } 148 149 s3Log.Debug(params) 150 151 req, resp := s.PutObjectRequest(params) 152 req.Handlers.Sign.Clear() 153 req.HTTPRequest.URL, _ = url.Parse(*param.Commit.UploadId) 154 155 start := totalSize - param.Size 156 end := totalSize - 1 157 var size string 158 if last { 159 size = strconv.FormatUint(totalSize, 10) 160 } else { 161 size = "*" 162 } 163 164 contentRange := fmt.Sprintf("bytes %v-%v/%v", start, end, size) 165 166 req.HTTPRequest.Header.Set("Content-Length", strconv.FormatUint(param.Size, 10)) 167 req.HTTPRequest.Header.Set("Content-Range", contentRange) 168 169 err = req.Send() 170 if err != nil { 171 // status indicating that we need more parts to finish this 172 if req.HTTPResponse.StatusCode == 308 { 173 err = nil 174 } else { 175 err = mapAwsError(err) 176 return 177 } 178 } 179 180 etag = resp.ETag 181 182 return 183 } 184 185 func (s *GCS3) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error) { 186 var commitData *GCSMultipartBlobCommitInput 187 var ok bool 188 if commitData, ok = param.Commit.backendData.(*GCSMultipartBlobCommitInput); !ok { 189 panic("Incorrect commit data type") 190 } 191 192 if commitData.Prev != nil { 193 if commitData.Prev.Size == 0 || commitData.Prev.Size%(256*1024) != 0 { 194 s3Log.Errorf("size of each block must be multiple of 256KB: %v", param.Size) 195 return nil, fuse.EINVAL 196 } 197 198 _, err := s.uploadPart(commitData.Prev, commitData.Size, false) 199 if err != nil { 200 return nil, err 201 } 202 } 203 commitData.Size += param.Size 204 205 copy := *param 206 commitData.Prev = © 207 param.Body = nil 208 209 return &MultipartBlobAddOutput{}, nil 210 } 211 212 func (s *GCS3) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error) { 213 var commitData *GCSMultipartBlobCommitInput 214 var ok bool 215 if commitData, ok = param.backendData.(*GCSMultipartBlobCommitInput); !ok { 216 panic("Incorrect commit data type") 217 } 218 219 if commitData.Prev == nil { 220 panic("commit should include last part") 221 } 222 223 etag, err := s.uploadPart(commitData.Prev, commitData.Size, true) 224 if err != nil { 225 return nil, err 226 } 227 228 return &MultipartBlobCommitOutput{ 229 ETag: etag, 230 }, nil 231 }