github.com/StarfishStorage/goofys@v0.23.2-0.20200415030923-535558486b34/internal/backend.go (about) 1 // Copyright 2019 Ka-Hing Cheung 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package internal 16 17 import ( 18 "fmt" 19 "io" 20 "io/ioutil" 21 "strings" 22 "sync" 23 "time" 24 25 "github.com/jacobsa/fuse" 26 ) 27 28 type Capabilities struct { 29 NoParallelMultipart bool 30 MaxMultipartSize uint64 31 // indicates that the blob store has native support for directories 32 DirBlob bool 33 Name string 34 } 35 36 type HeadBlobInput struct { 37 Key string 38 } 39 40 type BlobItemOutput struct { 41 Key *string 42 ETag *string 43 LastModified *time.Time 44 Size uint64 45 StorageClass *string 46 } 47 48 type HeadBlobOutput struct { 49 BlobItemOutput 50 51 ContentType *string 52 Metadata map[string]*string 53 IsDirBlob bool 54 55 RequestId string 56 } 57 58 type ListBlobsInput struct { 59 Prefix *string 60 Delimiter *string 61 MaxKeys *uint32 62 StartAfter *string // XXX: not supported by Azure 63 ContinuationToken *string 64 } 65 66 type BlobPrefixOutput struct { 67 Prefix *string 68 } 69 70 type ListBlobsOutput struct { 71 Prefixes []BlobPrefixOutput 72 Items []BlobItemOutput 73 NextContinuationToken *string 74 IsTruncated bool 75 76 RequestId string 77 } 78 79 type DeleteBlobInput struct { 80 Key string 81 } 82 83 type DeleteBlobOutput struct { 84 RequestId string 85 } 86 87 type DeleteBlobsInput struct { 88 Items []string 89 } 90 91 type DeleteBlobsOutput struct { 92 RequestId string 93 } 94 95 type RenameBlobInput struct { 96 Source string 97 Destination string 98 } 99 100 type RenameBlobOutput struct { 101 RequestId string 102 } 103 104 type CopyBlobInput struct { 105 Source string 106 Destination string 107 108 Size *uint64 109 ETag *string // if non-nil, do conditional copy 110 Metadata map[string]*string // if nil, copy from Source 111 StorageClass *string // if nil, copy from Source 112 } 113 114 type CopyBlobOutput struct { 115 RequestId string 116 } 117 118 type GetBlobInput struct { 119 Key string 120 Start uint64 121 Count uint64 122 IfMatch *string 123 } 124 125 type GetBlobOutput struct { 126 HeadBlobOutput 127 128 Body io.ReadCloser 129 130 RequestId string 131 } 132 133 type PutBlobInput struct { 134 Key string 135 Metadata map[string]*string 136 ContentType *string 137 DirBlob bool 138 139 Body io.ReadSeeker 140 Size *uint64 141 } 142 143 type PutBlobOutput struct { 144 ETag *string 145 StorageClass *string 146 147 RequestId string 148 } 149 150 type MultipartBlobBeginInput struct { 151 Key string 152 Metadata map[string]*string 153 ContentType *string 154 } 155 156 type MultipartBlobCommitInput struct { 157 Key *string 158 159 Metadata map[string]*string 160 UploadId *string 161 Parts []*string 162 NumParts uint32 163 164 // for GCS 165 backendData interface{} 166 } 167 168 type MultipartBlobAddInput struct { 169 Commit *MultipartBlobCommitInput 170 PartNumber uint32 171 172 Body io.ReadSeeker 173 174 Size uint64 // GCS wants to know part size 175 Last bool // GCS needs to know if this part is the last one 176 Offset uint64 // ADLv2 needs to know offset 177 } 178 179 type MultipartBlobAddOutput struct { 180 RequestId string 181 } 182 183 type MultipartBlobCommitOutput struct { 184 ETag *string 185 RequestId string 186 } 187 188 type MultipartBlobAbortOutput struct { 189 RequestId string 190 } 191 192 type MultipartExpireInput struct { 193 } 194 195 type MultipartExpireOutput struct { 196 RequestId string 197 } 198 199 type RemoveBucketInput struct { 200 } 201 202 type RemoveBucketOutput struct { 203 RequestId string 204 } 205 206 type MakeBucketInput struct { 207 } 208 209 type MakeBucketOutput struct { 210 RequestId string 211 } 212 213 /// Implementations of all the functions here are expected to be 214 /// concurrency-safe, except for 215 /// 216 /// Init() is called exactly once before any other functions are 217 /// called. 218 /// 219 /// Capabilities()/Bucket() are expected to be const 220 type StorageBackend interface { 221 Init(key string) error 222 Capabilities() *Capabilities 223 // typically this would return bucket/prefix 224 Bucket() string 225 HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) 226 ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) 227 DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error) 228 DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error) 229 RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error) 230 CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) 231 GetBlob(param *GetBlobInput) (*GetBlobOutput, error) 232 PutBlob(param *PutBlobInput) (*PutBlobOutput, error) 233 MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) 234 MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error) 235 MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error) 236 MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error) 237 MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error) 238 RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error) 239 MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error) 240 Delegate() interface{} 241 } 242 243 type Delegator interface { 244 Delegate() interface{} 245 } 246 247 var SmallActionsGate = Ticket{Total: 100}.Init() 248 249 type sortBlobPrefixOutput []BlobPrefixOutput 250 251 func (p sortBlobPrefixOutput) Len() int { 252 return len(p) 253 } 254 255 func (p sortBlobPrefixOutput) Less(i, j int) bool { 256 return *p[i].Prefix < *p[j].Prefix 257 } 258 259 func (p sortBlobPrefixOutput) Swap(i, j int) { 260 p[i], p[j] = p[j], p[i] 261 } 262 263 type sortBlobItemOutput []BlobItemOutput 264 265 func (p sortBlobItemOutput) Len() int { 266 return len(p) 267 } 268 269 func (p sortBlobItemOutput) Less(i, j int) bool { 270 return *p[i].Key < *p[j].Key 271 } 272 273 func (p sortBlobItemOutput) Swap(i, j int) { 274 p[i], p[j] = p[j], p[i] 275 } 276 277 func (b BlobItemOutput) String() string { 278 return fmt.Sprintf("%v: %v", *b.Key, b.Size) 279 } 280 281 func (b BlobPrefixOutput) String() string { 282 return fmt.Sprintf("%v", *b.Prefix) 283 } 284 285 type ReadSeekerCloser struct { 286 io.ReadSeeker 287 } 288 289 func (r *ReadSeekerCloser) Close() error { 290 if closer, ok := r.ReadSeeker.(io.Closer); ok { 291 return closer.Close() 292 } else { 293 return nil 294 } 295 } 296 297 type StorageBackendInitWrapper struct { 298 StorageBackend 299 init sync.Once 300 initKey string 301 initErr error 302 } 303 304 const INIT_ERR_BLOB = "mount.err" 305 306 func (s *StorageBackendInitWrapper) Init(key string) error { 307 s.init.Do(func() { 308 s.initErr = s.StorageBackend.Init(s.initKey) 309 if s.initErr != nil { 310 log.Errorf("%T Init: %v", s.StorageBackend, s.initErr) 311 s.StorageBackend = StorageBackendInitError{ 312 s.initErr, 313 *s.StorageBackend.Capabilities(), 314 } 315 } 316 }) 317 return s.initErr 318 } 319 320 func (s *StorageBackendInitWrapper) Capabilities() *Capabilities { 321 return s.StorageBackend.Capabilities() 322 } 323 324 func (s *StorageBackendInitWrapper) Bucket() string { 325 return s.StorageBackend.Bucket() 326 } 327 328 func (s *StorageBackendInitWrapper) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) { 329 s.Init("") 330 return s.StorageBackend.HeadBlob(param) 331 } 332 333 func (s *StorageBackendInitWrapper) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { 334 s.Init("") 335 return s.StorageBackend.ListBlobs(param) 336 } 337 338 func (s *StorageBackendInitWrapper) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error) { 339 s.Init("") 340 return s.StorageBackend.DeleteBlob(param) 341 } 342 343 func (s *StorageBackendInitWrapper) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error) { 344 s.Init("") 345 return s.StorageBackend.DeleteBlobs(param) 346 } 347 348 func (s *StorageBackendInitWrapper) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error) { 349 s.Init("") 350 return s.StorageBackend.RenameBlob(param) 351 } 352 353 func (s *StorageBackendInitWrapper) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { 354 s.Init("") 355 return s.StorageBackend.CopyBlob(param) 356 } 357 358 func (s *StorageBackendInitWrapper) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) { 359 s.Init("") 360 return s.StorageBackend.GetBlob(param) 361 } 362 363 func (s *StorageBackendInitWrapper) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { 364 s.Init("") 365 return s.StorageBackend.PutBlob(param) 366 } 367 368 func (s *StorageBackendInitWrapper) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) { 369 s.Init("") 370 return s.StorageBackend.MultipartBlobBegin(param) 371 } 372 373 func (s *StorageBackendInitWrapper) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error) { 374 s.Init("") 375 return s.StorageBackend.MultipartBlobAdd(param) 376 } 377 378 func (s *StorageBackendInitWrapper) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error) { 379 s.Init("") 380 return s.StorageBackend.MultipartBlobAbort(param) 381 } 382 383 func (s *StorageBackendInitWrapper) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error) { 384 s.Init("") 385 return s.StorageBackend.MultipartBlobCommit(param) 386 } 387 388 func (s *StorageBackendInitWrapper) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error) { 389 s.Init("") 390 return s.StorageBackend.MultipartExpire(param) 391 } 392 393 func (s *StorageBackendInitWrapper) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error) { 394 s.Init("") 395 return s.StorageBackend.RemoveBucket(param) 396 } 397 398 func (s *StorageBackendInitWrapper) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error) { 399 s.Init("") 400 return s.StorageBackend.MakeBucket(param) 401 } 402 403 type StorageBackendInitError struct { 404 error 405 cap Capabilities 406 } 407 408 func (e StorageBackendInitError) Init(key string) error { 409 return e 410 } 411 412 func (e StorageBackendInitError) Delegate() interface{} { 413 return e 414 } 415 416 func (e StorageBackendInitError) Capabilities() *Capabilities { 417 return &e.cap 418 } 419 420 func (s StorageBackendInitError) Bucket() string { 421 return "" 422 } 423 424 func (e StorageBackendInitError) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) { 425 if param.Key == INIT_ERR_BLOB { 426 return &HeadBlobOutput{ 427 BlobItemOutput: BlobItemOutput{ 428 Key: ¶m.Key, 429 Size: uint64(len(e.Error())), 430 LastModified: PTime(time.Now()), 431 }, 432 ContentType: PString("text/plain"), 433 }, nil 434 } else { 435 return nil, fuse.ENOENT 436 } 437 } 438 439 func (e StorageBackendInitError) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { 440 // return a fake blob 441 if param.Prefix == nil || *param.Prefix == "" { 442 return &ListBlobsOutput{ 443 Items: []BlobItemOutput{ 444 BlobItemOutput{ 445 Key: PString(INIT_ERR_BLOB), 446 Size: uint64(len(e.Error())), 447 LastModified: PTime(time.Now()), 448 }, 449 }, 450 }, nil 451 } else { 452 return &ListBlobsOutput{}, nil 453 } 454 } 455 456 func (e StorageBackendInitError) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error) { 457 return nil, e 458 } 459 460 func (e StorageBackendInitError) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error) { 461 return nil, e 462 } 463 464 func (e StorageBackendInitError) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error) { 465 return nil, e 466 } 467 468 func (e StorageBackendInitError) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { 469 return nil, e 470 } 471 472 func (e StorageBackendInitError) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) { 473 if param.Key == INIT_ERR_BLOB { 474 errStr := e.Error() 475 return &GetBlobOutput{ 476 HeadBlobOutput: HeadBlobOutput{ 477 BlobItemOutput: BlobItemOutput{ 478 Key: ¶m.Key, 479 Size: uint64(len(errStr)), 480 LastModified: PTime(time.Now()), 481 }, 482 ContentType: PString("text/plain"), 483 }, 484 Body: ioutil.NopCloser(strings.NewReader(errStr)), 485 }, nil 486 } else { 487 return nil, fuse.ENOENT 488 } 489 } 490 491 func (e StorageBackendInitError) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { 492 return nil, e 493 } 494 495 func (e StorageBackendInitError) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) { 496 return nil, e 497 } 498 499 func (e StorageBackendInitError) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error) { 500 return nil, e 501 } 502 503 func (e StorageBackendInitError) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error) { 504 return nil, e 505 } 506 507 func (e StorageBackendInitError) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error) { 508 return nil, e 509 } 510 511 func (e StorageBackendInitError) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error) { 512 return nil, e 513 } 514 515 func (e StorageBackendInitError) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error) { 516 return nil, e 517 } 518 519 func (e StorageBackendInitError) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error) { 520 return nil, e 521 }