github.com/parlaylabs/goofys@v0.24.1-0.20210317165919-e4fff29f7929/internal/backend.go (about) 1 // Copyright 2019 Ka-Hing Cheung 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package internal 16 17 import ( 18 "fmt" 19 "io" 20 "io/ioutil" 21 "strings" 22 "sync" 23 "time" 24 25 "github.com/jacobsa/fuse" 26 ) 27 28 type Capabilities struct { 29 NoParallelMultipart bool 30 MaxMultipartSize uint64 31 // indicates that the blob store has native support for directories 32 DirBlob bool 33 Name string 34 } 35 36 type HeadBlobInput struct { 37 Key string 38 } 39 40 type BlobItemOutput struct { 41 Key *string 42 ETag *string 43 LastModified *time.Time 44 Size uint64 45 StorageClass *string 46 } 47 48 type HeadBlobOutput struct { 49 BlobItemOutput 50 51 ContentType *string 52 Metadata map[string]*string 53 IsDirBlob bool 54 55 RequestId string 56 } 57 58 type ListBlobsInput struct { 59 Prefix *string 60 Delimiter *string 61 MaxKeys *uint32 62 StartAfter *string // XXX: not supported by Azure 63 ContinuationToken *string 64 } 65 66 type BlobPrefixOutput struct { 67 Prefix *string 68 } 69 70 type ListBlobsOutput struct { 71 Prefixes []BlobPrefixOutput 72 Items []BlobItemOutput 73 NextContinuationToken *string 74 IsTruncated bool 75 76 RequestId string 77 } 78 79 type DeleteBlobInput struct { 80 Key string 81 } 82 83 type DeleteBlobOutput struct { 84 RequestId string 85 } 86 87 type DeleteBlobsInput struct { 88 Items []string 89 } 90 91 type DeleteBlobsOutput struct { 92 RequestId string 93 } 94 95 type RenameBlobInput struct { 96 Source string 97 Destination string 98 } 99 100 type RenameBlobOutput struct { 101 RequestId string 102 } 103 104 type CopyBlobInput struct { 105 Source string 106 Destination string 107 108 Size *uint64 109 ETag *string // if non-nil, do conditional copy 110 Metadata map[string]*string // if nil, copy from Source 111 StorageClass *string // if nil, copy from Source 112 } 113 114 type CopyBlobOutput struct { 115 RequestId string 116 } 117 118 type GetBlobInput struct { 119 Key string 120 Start uint64 121 Count uint64 122 IfMatch *string 123 } 124 125 type GetBlobOutput struct { 126 HeadBlobOutput 127 128 Body io.ReadCloser 129 130 RequestId string 131 } 132 133 type PutBlobInput struct { 134 Key string 135 Metadata map[string]*string 136 ContentType *string 137 DirBlob bool 138 139 Body io.ReadSeeker 140 Size *uint64 141 } 142 143 type PutBlobOutput struct { 144 ETag *string 145 LastModified *time.Time 146 StorageClass *string 147 148 RequestId string 149 } 150 151 type MultipartBlobBeginInput struct { 152 Key string 153 Metadata map[string]*string 154 ContentType *string 155 } 156 157 type MultipartBlobCommitInput struct { 158 Key *string 159 160 Metadata map[string]*string 161 UploadId *string 162 Parts []*string 163 NumParts uint32 164 165 // for GCS 166 backendData interface{} 167 } 168 169 type MultipartBlobAddInput struct { 170 Commit *MultipartBlobCommitInput 171 PartNumber uint32 172 173 Body io.ReadSeeker 174 175 Size uint64 // GCS wants to know part size 176 Last bool // GCS needs to know if this part is the last one 177 Offset uint64 // ADLv2 needs to know offset 178 } 179 180 type MultipartBlobAddOutput struct { 181 RequestId string 182 } 183 184 type MultipartBlobCommitOutput struct { 185 ETag *string 186 LastModified *time.Time 187 StorageClass *string 188 189 RequestId string 190 } 191 192 type MultipartBlobAbortOutput struct { 193 RequestId string 194 } 195 196 type MultipartExpireInput struct { 197 } 198 199 type MultipartExpireOutput struct { 200 RequestId string 201 } 202 203 type RemoveBucketInput struct { 204 } 205 206 type RemoveBucketOutput struct { 207 RequestId string 208 } 209 210 type MakeBucketInput struct { 211 } 212 213 type MakeBucketOutput struct { 214 RequestId string 215 } 216 217 /// Implementations of all the functions here are expected to be 218 /// concurrency-safe, except for 219 /// 220 /// Init() is called exactly once before any other functions are 221 /// called. 222 /// 223 /// Capabilities()/Bucket() are expected to be const 224 type StorageBackend interface { 225 Init(key string) error 226 Capabilities() *Capabilities 227 // typically this would return bucket/prefix 228 Bucket() string 229 HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) 230 ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) 231 DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error) 232 DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error) 233 RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error) 234 CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) 235 GetBlob(param *GetBlobInput) (*GetBlobOutput, error) 236 PutBlob(param *PutBlobInput) (*PutBlobOutput, error) 237 MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) 238 MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error) 239 MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error) 240 MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error) 241 MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error) 242 RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error) 243 MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error) 244 Delegate() interface{} 245 } 246 247 type Delegator interface { 248 Delegate() interface{} 249 } 250 251 var SmallActionsGate = Ticket{Total: 100}.Init() 252 253 type sortBlobPrefixOutput []BlobPrefixOutput 254 255 func (p sortBlobPrefixOutput) Len() int { 256 return len(p) 257 } 258 259 func (p sortBlobPrefixOutput) Less(i, j int) bool { 260 return *p[i].Prefix < *p[j].Prefix 261 } 262 263 func (p sortBlobPrefixOutput) Swap(i, j int) { 264 p[i], p[j] = p[j], p[i] 265 } 266 267 type sortBlobItemOutput []BlobItemOutput 268 269 func (p sortBlobItemOutput) Len() int { 270 return len(p) 271 } 272 273 func (p sortBlobItemOutput) Less(i, j int) bool { 274 return *p[i].Key < *p[j].Key 275 } 276 277 func (p sortBlobItemOutput) Swap(i, j int) { 278 p[i], p[j] = p[j], p[i] 279 } 280 281 func (b BlobItemOutput) String() string { 282 return fmt.Sprintf("%v: %v", *b.Key, b.Size) 283 } 284 285 func (b BlobPrefixOutput) String() string { 286 return fmt.Sprintf("%v", *b.Prefix) 287 } 288 289 type ReadSeekerCloser struct { 290 io.ReadSeeker 291 } 292 293 func (r *ReadSeekerCloser) Close() error { 294 if closer, ok := r.ReadSeeker.(io.Closer); ok { 295 return closer.Close() 296 } else { 297 return nil 298 } 299 } 300 301 type StorageBackendInitWrapper struct { 302 StorageBackend 303 init sync.Once 304 initKey string 305 initErr error 306 } 307 308 const INIT_ERR_BLOB = "mount.err" 309 310 func (s *StorageBackendInitWrapper) Init(key string) error { 311 s.init.Do(func() { 312 s.initErr = s.StorageBackend.Init(s.initKey) 313 if s.initErr != nil { 314 log.Errorf("%T Init: %v", s.StorageBackend, s.initErr) 315 s.StorageBackend = StorageBackendInitError{ 316 s.initErr, 317 *s.StorageBackend.Capabilities(), 318 } 319 } 320 }) 321 return s.initErr 322 } 323 324 func (s *StorageBackendInitWrapper) Capabilities() *Capabilities { 325 return s.StorageBackend.Capabilities() 326 } 327 328 func (s *StorageBackendInitWrapper) Bucket() string { 329 return s.StorageBackend.Bucket() 330 } 331 332 func (s *StorageBackendInitWrapper) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) { 333 s.Init("") 334 return s.StorageBackend.HeadBlob(param) 335 } 336 337 func (s *StorageBackendInitWrapper) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { 338 s.Init("") 339 return s.StorageBackend.ListBlobs(param) 340 } 341 342 func (s *StorageBackendInitWrapper) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error) { 343 s.Init("") 344 return s.StorageBackend.DeleteBlob(param) 345 } 346 347 func (s *StorageBackendInitWrapper) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error) { 348 s.Init("") 349 return s.StorageBackend.DeleteBlobs(param) 350 } 351 352 func (s *StorageBackendInitWrapper) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error) { 353 s.Init("") 354 return s.StorageBackend.RenameBlob(param) 355 } 356 357 func (s *StorageBackendInitWrapper) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { 358 s.Init("") 359 return s.StorageBackend.CopyBlob(param) 360 } 361 362 func (s *StorageBackendInitWrapper) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) { 363 s.Init("") 364 return s.StorageBackend.GetBlob(param) 365 } 366 367 func (s *StorageBackendInitWrapper) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { 368 s.Init("") 369 return s.StorageBackend.PutBlob(param) 370 } 371 372 func (s *StorageBackendInitWrapper) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) { 373 s.Init("") 374 return s.StorageBackend.MultipartBlobBegin(param) 375 } 376 377 func (s *StorageBackendInitWrapper) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error) { 378 s.Init("") 379 return s.StorageBackend.MultipartBlobAdd(param) 380 } 381 382 func (s *StorageBackendInitWrapper) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error) { 383 s.Init("") 384 return s.StorageBackend.MultipartBlobAbort(param) 385 } 386 387 func (s *StorageBackendInitWrapper) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error) { 388 s.Init("") 389 return s.StorageBackend.MultipartBlobCommit(param) 390 } 391 392 func (s *StorageBackendInitWrapper) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error) { 393 s.Init("") 394 return s.StorageBackend.MultipartExpire(param) 395 } 396 397 func (s *StorageBackendInitWrapper) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error) { 398 s.Init("") 399 return s.StorageBackend.RemoveBucket(param) 400 } 401 402 func (s *StorageBackendInitWrapper) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error) { 403 s.Init("") 404 return s.StorageBackend.MakeBucket(param) 405 } 406 407 type StorageBackendInitError struct { 408 error 409 cap Capabilities 410 } 411 412 func (e StorageBackendInitError) Init(key string) error { 413 return e 414 } 415 416 func (e StorageBackendInitError) Delegate() interface{} { 417 return e 418 } 419 420 func (e StorageBackendInitError) Capabilities() *Capabilities { 421 return &e.cap 422 } 423 424 func (s StorageBackendInitError) Bucket() string { 425 return "" 426 } 427 428 func (e StorageBackendInitError) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) { 429 if param.Key == INIT_ERR_BLOB { 430 return &HeadBlobOutput{ 431 BlobItemOutput: BlobItemOutput{ 432 Key: ¶m.Key, 433 Size: uint64(len(e.Error())), 434 LastModified: PTime(time.Now()), 435 }, 436 ContentType: PString("text/plain"), 437 }, nil 438 } else { 439 return nil, fuse.ENOENT 440 } 441 } 442 443 func (e StorageBackendInitError) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { 444 // return a fake blob 445 if param.Prefix == nil || *param.Prefix == "" { 446 return &ListBlobsOutput{ 447 Items: []BlobItemOutput{ 448 BlobItemOutput{ 449 Key: PString(INIT_ERR_BLOB), 450 Size: uint64(len(e.Error())), 451 LastModified: PTime(time.Now()), 452 }, 453 }, 454 }, nil 455 } else { 456 return &ListBlobsOutput{}, nil 457 } 458 } 459 460 func (e StorageBackendInitError) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error) { 461 return nil, e 462 } 463 464 func (e StorageBackendInitError) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error) { 465 return nil, e 466 } 467 468 func (e StorageBackendInitError) RenameBlob(param *RenameBlobInput) (*RenameBlobOutput, error) { 469 return nil, e 470 } 471 472 func (e StorageBackendInitError) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { 473 return nil, e 474 } 475 476 func (e StorageBackendInitError) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) { 477 if param.Key == INIT_ERR_BLOB { 478 errStr := e.Error() 479 return &GetBlobOutput{ 480 HeadBlobOutput: HeadBlobOutput{ 481 BlobItemOutput: BlobItemOutput{ 482 Key: ¶m.Key, 483 Size: uint64(len(errStr)), 484 LastModified: PTime(time.Now()), 485 }, 486 ContentType: PString("text/plain"), 487 }, 488 Body: ioutil.NopCloser(strings.NewReader(errStr)), 489 }, nil 490 } else { 491 return nil, fuse.ENOENT 492 } 493 } 494 495 func (e StorageBackendInitError) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { 496 return nil, e 497 } 498 499 func (e StorageBackendInitError) MultipartBlobBegin(param *MultipartBlobBeginInput) (*MultipartBlobCommitInput, error) { 500 return nil, e 501 } 502 503 func (e StorageBackendInitError) MultipartBlobAdd(param *MultipartBlobAddInput) (*MultipartBlobAddOutput, error) { 504 return nil, e 505 } 506 507 func (e StorageBackendInitError) MultipartBlobAbort(param *MultipartBlobCommitInput) (*MultipartBlobAbortOutput, error) { 508 return nil, e 509 } 510 511 func (e StorageBackendInitError) MultipartBlobCommit(param *MultipartBlobCommitInput) (*MultipartBlobCommitOutput, error) { 512 return nil, e 513 } 514 515 func (e StorageBackendInitError) MultipartExpire(param *MultipartExpireInput) (*MultipartExpireOutput, error) { 516 return nil, e 517 } 518 519 func (e StorageBackendInitError) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error) { 520 return nil, e 521 } 522 523 func (e StorageBackendInitError) MakeBucket(param *MakeBucketInput) (*MakeBucketOutput, error) { 524 return nil, e 525 }