launchpad.net/~rogpeppe/juju-core/500-errgo-fix@v0.0.0-20140213181702-000000002356/provider/ec2/storage.go (about) 1 // Copyright 2012, 2013 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 package ec2 5 6 import ( 7 "fmt" 8 "io" 9 "net" 10 "sync" 11 "time" 12 13 "launchpad.net/goamz/s3" 14 15 "launchpad.net/juju-core/environs/storage" 16 "launchpad.net/juju-core/errors" 17 "launchpad.net/juju-core/utils" 18 ) 19 20 func init() { 21 // We will decide when to retry and under what circumstances, not s3. 22 // Sometimes it is expected a file may not exist and we don't want s3 23 // to hold things up by unilaterally deciding to retry for no good reason. 24 s3.RetryAttempts(false) 25 } 26 27 func NewStorage(bucket *s3.Bucket) storage.Storage { 28 return &ec2storage{bucket: bucket} 29 } 30 31 // ec2storage implements storage.Storage on 32 // an ec2.bucket. 33 type ec2storage struct { 34 sync.Mutex 35 madeBucket bool 36 bucket *s3.Bucket 37 } 38 39 // makeBucket makes the environent's control bucket, the 40 // place where bootstrap information and deployed charms 41 // are stored. To avoid two round trips on every PUT operation, 42 // we do this only once for each environ. 43 func (s *ec2storage) makeBucket() error { 44 s.Lock() 45 defer s.Unlock() 46 if s.madeBucket { 47 return nil 48 } 49 // PutBucket always return a 200 if we recreate an existing bucket for the 50 // original s3.amazonaws.com endpoint. For all other endpoints PutBucket 51 // returns 409 with a known subcode. 52 if err := s.bucket.PutBucket(s3.Private); err != nil && s3ErrCode(err) != "BucketAlreadyOwnedByYou" { 53 return err 54 } 55 56 s.madeBucket = true 57 return nil 58 } 59 60 func (s *ec2storage) Put(file string, r io.Reader, length int64) error { 61 if err := s.makeBucket(); err != nil { 62 return fmt.Errorf("cannot make S3 control bucket: %v", err) 63 } 64 err := s.bucket.PutReader(file, r, length, "binary/octet-stream", s3.Private) 65 if err != nil { 66 return fmt.Errorf("cannot write file %q to control bucket: %v", file, err) 67 } 68 return nil 69 } 70 71 func (s *ec2storage) Get(file string) (r io.ReadCloser, err error) { 72 r, err = s.bucket.GetReader(file) 73 return r, maybeNotFound(err) 74 } 75 76 func (s *ec2storage) URL(name string) (string, error) { 77 // 10 years should be good enough. 78 return s.bucket.SignedURL(name, time.Now().AddDate(10, 0, 0)), nil 79 } 80 81 var storageAttempt = utils.AttemptStrategy{ 82 Total: 5 * time.Second, 83 Delay: 200 * time.Millisecond, 84 } 85 86 // ConsistencyStrategy is specified in the StorageReader interface. 87 func (s *ec2storage) DefaultConsistencyStrategy() utils.AttemptStrategy { 88 return storageAttempt 89 } 90 91 // ShouldRetry is specified in the StorageReader interface. 92 func (s *ec2storage) ShouldRetry(err error) bool { 93 if err == nil { 94 return false 95 } 96 switch err { 97 case io.ErrUnexpectedEOF, io.EOF: 98 return true 99 } 100 if s3ErrorStatusCode(err) == 404 { 101 return true 102 } 103 switch e := err.(type) { 104 case *net.DNSError: 105 return true 106 case *net.OpError: 107 switch e.Op { 108 case "read", "write": 109 return true 110 } 111 case *s3.Error: 112 switch e.Code { 113 case "InternalError": 114 return true 115 } 116 } 117 return false 118 } 119 120 // s3ErrorStatusCode returns the HTTP status of the S3 request error, 121 // if it is an error from an S3 operation, or 0 if it was not. 122 func s3ErrorStatusCode(err error) int { 123 if err, _ := err.(*s3.Error); err != nil { 124 return err.StatusCode 125 } 126 return 0 127 } 128 129 // s3ErrCode returns the text status code of the S3 error code. 130 func s3ErrCode(err error) string { 131 if err, ok := err.(*s3.Error); ok { 132 return err.Code 133 } 134 return "" 135 } 136 137 func (s *ec2storage) Remove(file string) error { 138 err := s.bucket.Del(file) 139 // If we can't delete the object because the bucket doesn't 140 // exist, then we don't care. 141 if s3ErrorStatusCode(err) == 404 { 142 return nil 143 } 144 return err 145 } 146 147 func (s *ec2storage) List(prefix string) ([]string, error) { 148 // TODO cope with more than 1000 objects in the bucket. 149 resp, err := s.bucket.List(prefix, "", "", 0) 150 if err != nil { 151 // If the bucket is not found, it's not an error 152 // because it's only created when the first 153 // file is put. 154 if s3ErrorStatusCode(err) == 404 { 155 return nil, nil 156 } 157 return nil, err 158 } 159 var names []string 160 for _, key := range resp.Contents { 161 names = append(names, key.Key) 162 } 163 return names, nil 164 } 165 166 func (s *ec2storage) RemoveAll() error { 167 names, err := storage.List(s, "") 168 if err != nil { 169 return err 170 } 171 // Remove all the objects in parallel to minimize round-trips. 172 // If we're in danger of having hundreds of objects, 173 // we'll want to change this to limit the number 174 // of concurrent operations. 175 var wg sync.WaitGroup 176 wg.Add(len(names)) 177 errc := make(chan error, len(names)) 178 for _, name := range names { 179 name := name 180 go func() { 181 if err := s.Remove(name); err != nil { 182 errc <- err 183 } 184 wg.Done() 185 }() 186 } 187 wg.Wait() 188 select { 189 case err := <-errc: 190 return fmt.Errorf("cannot delete all provider state: %v", err) 191 default: 192 } 193 194 s.Lock() 195 defer s.Unlock() 196 // Even DelBucket fails, it won't harm if we try again - the operation 197 // might have succeeded even if we get an error. 198 s.madeBucket = false 199 err = deleteBucket(s) 200 err = s.bucket.DelBucket() 201 if s3ErrorStatusCode(err) == 404 { 202 return nil 203 } 204 return err 205 } 206 207 func deleteBucket(s *ec2storage) (err error) { 208 for a := s.DefaultConsistencyStrategy().Start(); a.Next(); { 209 err = s.bucket.DelBucket() 210 if err == nil || !s.ShouldRetry(err) { 211 break 212 } 213 } 214 return err 215 } 216 217 func maybeNotFound(err error) error { 218 if err != nil && s3ErrorStatusCode(err) == 404 { 219 return errors.NewNotFoundError(err, "") 220 } 221 return err 222 }