github.com/djmaze/goofys@v0.24.2/internal/aws_test.go (about) 1 // Copyright 2016 Ka-Hing Cheung 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package internal 16 17 import ( 18 . "github.com/djmaze/goofys/api/common" 19 . "gopkg.in/check.v1" 20 21 "fmt" 22 "sync" 23 "syscall" 24 "time" 25 ) 26 27 type AwsTest struct { 28 s3 *S3Backend 29 } 30 31 var _ = Suite(&AwsTest{}) 32 33 func (s *AwsTest) SetUpSuite(t *C) { 34 var err error 35 s.s3, err = NewS3("", &FlagStorage{}, &S3Config{ 36 Region: "us-east-1", 37 }) 38 t.Assert(err, IsNil) 39 } 40 41 func (s *AwsTest) TestRegionDetection(t *C) { 42 s.s3.bucket = "goofys-eu-west-1.kahing.xyz" 43 44 err, isAws := s.s3.detectBucketLocationByHEAD() 45 t.Assert(err, IsNil) 46 t.Assert(*s.s3.awsConfig.Region, Equals, "eu-west-1") 47 t.Assert(isAws, Equals, true) 48 } 49 50 func (s *AwsTest) TestBucket404(t *C) { 51 s.s3.bucket = RandStringBytesMaskImprSrc(64) 52 53 err, isAws := s.s3.detectBucketLocationByHEAD() 54 t.Assert(err, Equals, syscall.ENXIO) 55 t.Assert(isAws, Equals, true) 56 } 57 58 type S3BucketEventualConsistency struct { 59 *S3Backend 60 // a list of blobs ever put by this backend, we speculatively 61 // retry on these blobs to workaround eventual consistency 62 mu sync.RWMutex 63 blobs map[string]bool 64 } 65 66 func NewS3BucketEventualConsistency(s *S3Backend) *S3BucketEventualConsistency { 67 return &S3BucketEventualConsistency{ 68 s, 69 sync.RWMutex{}, 70 make(map[string]bool), 71 } 72 } 73 74 func (s *S3BucketEventualConsistency) Init(key string) (err error) { 75 // TODO: make Init return errno instead 76 NoSuchBucket := fmt.Sprintf("bucket %v does not exist", s.Bucket()) 77 78 for i := 0; i < 10; i++ { 79 err = s.S3Backend.Init(key) 80 if err != nil && err.Error() == NoSuchBucket { 81 s3Log.Infof("waiting for bucket") 82 time.Sleep((time.Duration(i) + 1) * 2 * time.Second) 83 } else { 84 return 85 } 86 } 87 88 return 89 } 90 91 func (s *S3BucketEventualConsistency) HeadBlob(param *HeadBlobInput) (*HeadBlobOutput, error) { 92 var err error 93 var res *HeadBlobOutput 94 95 for i := 0; i < 10; i++ { 96 res, err = s.S3Backend.HeadBlob(param) 97 switch err { 98 case syscall.ENXIO: 99 s3Log.Infof("waiting for bucket") 100 time.Sleep((time.Duration(i) + 1) * 2 * time.Second) 101 case syscall.ENOENT: 102 s.mu.RLock() 103 _, ok := s.blobs[param.Key] 104 s.mu.RUnlock() 105 106 if ok { 107 s3Log.Infof("waiting for blob: %v", param.Key) 108 time.Sleep((time.Duration(i) + 1) * 20 * time.Millisecond) 109 } 110 default: 111 return res, err 112 } 113 } 114 115 return res, err 116 } 117 118 func (s *S3BucketEventualConsistency) ListBlobs(param *ListBlobsInput) (*ListBlobsOutput, error) { 119 for i := 0; i < 10; i++ { 120 res, err := s.S3Backend.ListBlobs(param) 121 switch err { 122 case syscall.ENXIO: 123 s3Log.Infof("waiting for bucket") 124 time.Sleep((time.Duration(i) + 1) * 2 * time.Second) 125 default: 126 return res, err 127 } 128 } 129 130 return nil, syscall.ENXIO 131 } 132 133 func (s *S3BucketEventualConsistency) DeleteBlob(param *DeleteBlobInput) (*DeleteBlobOutput, error) { 134 for i := 0; i < 10; i++ { 135 res, err := s.S3Backend.DeleteBlob(param) 136 switch err { 137 case syscall.ENXIO: 138 s3Log.Infof("waiting for bucket") 139 time.Sleep((time.Duration(i) + 1) * 2 * time.Second) 140 default: 141 return res, err 142 } 143 } 144 145 return nil, syscall.ENXIO 146 } 147 148 func (s *S3BucketEventualConsistency) DeleteBlobs(param *DeleteBlobsInput) (*DeleteBlobsOutput, error) { 149 for i := 0; i < 10; i++ { 150 res, err := s.S3Backend.DeleteBlobs(param) 151 switch err { 152 case syscall.ENXIO: 153 s3Log.Infof("waiting for bucket") 154 time.Sleep((time.Duration(i) + 1) * 2 * time.Second) 155 default: 156 return res, err 157 } 158 } 159 160 return nil, syscall.ENXIO 161 } 162 163 func (s *S3BucketEventualConsistency) CopyBlob(param *CopyBlobInput) (*CopyBlobOutput, error) { 164 for i := 0; i < 10; i++ { 165 res, err := s.S3Backend.CopyBlob(param) 166 switch err { 167 case syscall.ENXIO: 168 s3Log.Infof("waiting for bucket") 169 time.Sleep((time.Duration(i) + 1) * 2 * time.Second) 170 default: 171 return res, err 172 } 173 } 174 175 return nil, syscall.ENXIO 176 } 177 178 func (s *S3BucketEventualConsistency) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { 179 s.mu.Lock() 180 s.blobs[param.Key] = true 181 s.mu.Unlock() 182 183 for i := 0; i < 10; i++ { 184 res, err := s.S3Backend.PutBlob(param) 185 switch err { 186 case syscall.ENXIO: 187 param.Body.Seek(0, 0) 188 s3Log.Infof("waiting for bucket") 189 time.Sleep((time.Duration(i) + 1) * 2 * time.Second) 190 default: 191 return res, err 192 } 193 } 194 195 return nil, syscall.ENXIO 196 } 197 198 func (s *S3BucketEventualConsistency) RemoveBucket(param *RemoveBucketInput) (*RemoveBucketOutput, error) { 199 for i := 0; i < 10; i++ { 200 res, err := s.S3Backend.RemoveBucket(param) 201 switch err { 202 case syscall.ENXIO: 203 s3Log.Infof("waiting for bucket") 204 time.Sleep((time.Duration(i) + 1) * 2 * time.Second) 205 default: 206 return res, err 207 } 208 } 209 210 return nil, syscall.ENXIO 211 }