github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/blobserver/s3/s3.go (about) 1 /* 2 Copyright 2011 Google Inc. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 /* 18 Package s3 registers the "s3" blobserver storage type, storing 19 blobs in an Amazon Web Services' S3 storage bucket. 20 21 Example low-level config: 22 23 "/r1/": { 24 "handler": "storage-s3", 25 "handlerArgs": { 26 "bucket": "foo", 27 "aws_access_key": "...", 28 "aws_secret_access_key": "...", 29 "skipStartupCheck": false 30 } 31 }, 32 33 */ 34 package s3 35 36 import ( 37 "fmt" 38 "log" 39 "strings" 40 41 "camlistore.org/pkg/blobserver" 42 "camlistore.org/pkg/fault" 43 "camlistore.org/pkg/jsonconfig" 44 "camlistore.org/pkg/misc/amazon/s3" 45 ) 46 47 var ( 48 faultReceive = fault.NewInjector("s3_receive") 49 faultEnumerate = fault.NewInjector("s3_enumerate") 50 faultStat = fault.NewInjector("s3_stat") 51 faultGet = fault.NewInjector("s3_get") 52 ) 53 54 type s3Storage struct { 55 s3Client *s3.Client 56 bucket string 57 hostname string 58 } 59 60 func (s *s3Storage) String() string { 61 return fmt.Sprintf("\"s3\" blob storage at host %q, bucket %q", s.hostname, s.bucket) 62 } 63 64 func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) { 65 hostname := config.OptionalString("hostname", "s3.amazonaws.com") 66 client := &s3.Client{ 67 Auth: &s3.Auth{ 68 AccessKey: config.RequiredString("aws_access_key"), 69 SecretAccessKey: config.RequiredString("aws_secret_access_key"), 70 Hostname: hostname, 71 }, 72 } 73 sto := &s3Storage{ 74 s3Client: client, 75 bucket: config.RequiredString("bucket"), 76 hostname: hostname, 77 } 78 skipStartupCheck := config.OptionalBool("skipStartupCheck", false) 79 if err := config.Validate(); err != nil { 80 return nil, err 81 } 82 if !skipStartupCheck { 83 _, err := client.ListBucket(sto.bucket, "", 1) 84 if serr, ok := err.(*s3.Error); ok { 85 if serr.AmazonCode == "NoSuchBucket" { 86 return nil, fmt.Errorf("Bucket %q doesn't exist.", sto.bucket) 87 } 88 89 // This code appears when the hostname has dots in it: 90 if serr.AmazonCode == "PermanentRedirect" { 91 loc, lerr := client.BucketLocation(sto.bucket) 92 if lerr != nil { 93 return nil, fmt.Errorf("Wrong server for bucket %q; and error determining bucket's location: %v", sto.bucket, lerr) 94 } 95 client.Auth.Hostname = loc 96 _, err = client.ListBucket(sto.bucket, "", 1) 97 if err == nil { 98 log.Printf("Warning: s3 server should be %q, not %q. Change config file to avoid start-up latency.", client.Auth.Hostname, hostname) 99 } 100 } 101 102 // This path occurs when the user set the 103 // wrong server, or didn't set one at all, but 104 // the bucket doesn't have dots in it: 105 if serr.UseEndpoint != "" { 106 // UseEndpoint will be e.g. "brads3test-ca.s3-us-west-1.amazonaws.com" 107 // But we only want the "s3-us-west-1.amazonaws.com" part. 108 client.Auth.Hostname = strings.TrimPrefix(serr.UseEndpoint, sto.bucket+".") 109 _, err = client.ListBucket(sto.bucket, "", 1) 110 if err == nil { 111 log.Printf("Warning: s3 server should be %q, not %q. Change config file to avoid start-up latency.", client.Auth.Hostname, hostname) 112 } 113 } 114 } 115 if err != nil { 116 return nil, fmt.Errorf("Error listing bucket %s: %v", sto.bucket, err) 117 } 118 } 119 return sto, nil 120 } 121 122 func init() { 123 blobserver.RegisterStorageConstructor("s3", blobserver.StorageConstructor(newFromConfig)) 124 }