github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/blobserver/s3/s3.go (about)

     1  /*
     2  Copyright 2011 Google Inc.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8       http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  /*
    18  Package s3 registers the "s3" blobserver storage type, storing
    19  blobs in an Amazon Web Services' S3 storage bucket.
    20  
    21  Example low-level config:
    22  
    23       "/r1/": {
    24           "handler": "storage-s3",
    25           "handlerArgs": {
    26              "bucket": "foo",
    27              "aws_access_key": "...",
    28              "aws_secret_access_key": "...",
    29              "skipStartupCheck": false
    30            }
    31       },
    32  
    33  */
    34  package s3
    35  
    36  import (
    37  	"fmt"
    38  	"log"
    39  	"strings"
    40  
    41  	"camlistore.org/pkg/blobserver"
    42  	"camlistore.org/pkg/fault"
    43  	"camlistore.org/pkg/jsonconfig"
    44  	"camlistore.org/pkg/misc/amazon/s3"
    45  )
    46  
    47  var (
    48  	faultReceive   = fault.NewInjector("s3_receive")
    49  	faultEnumerate = fault.NewInjector("s3_enumerate")
    50  	faultStat      = fault.NewInjector("s3_stat")
    51  	faultGet       = fault.NewInjector("s3_get")
    52  )
    53  
    54  type s3Storage struct {
    55  	s3Client *s3.Client
    56  	bucket   string
    57  	hostname string
    58  }
    59  
    60  func (s *s3Storage) String() string {
    61  	return fmt.Sprintf("\"s3\" blob storage at host %q, bucket %q", s.hostname, s.bucket)
    62  }
    63  
    64  func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
    65  	hostname := config.OptionalString("hostname", "s3.amazonaws.com")
    66  	client := &s3.Client{
    67  		Auth: &s3.Auth{
    68  			AccessKey:       config.RequiredString("aws_access_key"),
    69  			SecretAccessKey: config.RequiredString("aws_secret_access_key"),
    70  			Hostname:        hostname,
    71  		},
    72  	}
    73  	sto := &s3Storage{
    74  		s3Client: client,
    75  		bucket:   config.RequiredString("bucket"),
    76  		hostname: hostname,
    77  	}
    78  	skipStartupCheck := config.OptionalBool("skipStartupCheck", false)
    79  	if err := config.Validate(); err != nil {
    80  		return nil, err
    81  	}
    82  	if !skipStartupCheck {
    83  		_, err := client.ListBucket(sto.bucket, "", 1)
    84  		if serr, ok := err.(*s3.Error); ok {
    85  			if serr.AmazonCode == "NoSuchBucket" {
    86  				return nil, fmt.Errorf("Bucket %q doesn't exist.", sto.bucket)
    87  			}
    88  
    89  			// This code appears when the hostname has dots in it:
    90  			if serr.AmazonCode == "PermanentRedirect" {
    91  				loc, lerr := client.BucketLocation(sto.bucket)
    92  				if lerr != nil {
    93  					return nil, fmt.Errorf("Wrong server for bucket %q; and error determining bucket's location: %v", sto.bucket, lerr)
    94  				}
    95  				client.Auth.Hostname = loc
    96  				_, err = client.ListBucket(sto.bucket, "", 1)
    97  				if err == nil {
    98  					log.Printf("Warning: s3 server should be %q, not %q. Change config file to avoid start-up latency.", client.Auth.Hostname, hostname)
    99  				}
   100  			}
   101  
   102  			// This path occurs when the user set the
   103  			// wrong server, or didn't set one at all, but
   104  			// the bucket doesn't have dots in it:
   105  			if serr.UseEndpoint != "" {
   106  				// UseEndpoint will be e.g. "brads3test-ca.s3-us-west-1.amazonaws.com"
   107  				// But we only want the "s3-us-west-1.amazonaws.com" part.
   108  				client.Auth.Hostname = strings.TrimPrefix(serr.UseEndpoint, sto.bucket+".")
   109  				_, err = client.ListBucket(sto.bucket, "", 1)
   110  				if err == nil {
   111  					log.Printf("Warning: s3 server should be %q, not %q. Change config file to avoid start-up latency.", client.Auth.Hostname, hostname)
   112  				}
   113  			}
   114  		}
   115  		if err != nil {
   116  			return nil, fmt.Errorf("Error listing bucket %s: %v", sto.bucket, err)
   117  		}
   118  	}
   119  	return sto, nil
   120  }
   121  
   122  func init() {
   123  	blobserver.RegisterStorageConstructor("s3", blobserver.StorageConstructor(newFromConfig))
   124  }