github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/blobserver/google/cloudstorage/storage.go (about)

     1  /*
     2  Copyright 2011 Google Inc.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8       http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package cloudstorage registers the "googlecloudstorage" blob storage type, storing blobs
    18  // on Google Cloud Storage (not Google Drive).
    19  // See https://cloud.google.com/products/cloud-storage
    20  package cloudstorage
    21  
    22  import (
    23  	"bytes"
    24  	"errors"
    25  	"io"
    26  	"io/ioutil"
    27  	"log"
    28  
    29  	"camlistore.org/pkg/blob"
    30  	"camlistore.org/pkg/blobserver"
    31  	"camlistore.org/pkg/constants"
    32  	"camlistore.org/pkg/context"
    33  	"camlistore.org/pkg/googlestorage"
    34  	"camlistore.org/pkg/jsonconfig"
    35  )
    36  
    37  type Storage struct {
    38  	bucket string // the gs bucket containing blobs
    39  	client *googlestorage.Client
    40  }
    41  
    42  var _ blobserver.MaxEnumerateConfig = (*Storage)(nil)
    43  
    44  func (gs *Storage) MaxEnumerate() int { return 1000 }
    45  
    46  func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
    47  	auth := config.RequiredObject("auth")
    48  
    49  	gs := &Storage{
    50  		bucket: config.RequiredString("bucket"),
    51  		client: googlestorage.NewClient(googlestorage.MakeOauthTransport(
    52  			auth.RequiredString("client_id"),
    53  			auth.RequiredString("client_secret"),
    54  			auth.RequiredString("refresh_token"))),
    55  	}
    56  	if err := config.Validate(); err != nil {
    57  		return nil, err
    58  	}
    59  	if err := auth.Validate(); err != nil {
    60  		return nil, err
    61  	}
    62  	return gs, nil
    63  }
    64  
    65  func (gs *Storage) EnumerateBlobs(ctx *context.Context, dest chan<- blob.SizedRef, after string, limit int) error {
    66  	defer close(dest)
    67  	objs, err := gs.client.EnumerateObjects(gs.bucket, after, limit)
    68  	if err != nil {
    69  		log.Printf("gstorage EnumerateObjects: %v", err)
    70  		return err
    71  	}
    72  	for _, obj := range objs {
    73  		br, ok := blob.Parse(obj.Key)
    74  		if !ok {
    75  			continue
    76  		}
    77  		select {
    78  		case dest <- blob.SizedRef{Ref: br, Size: uint32(obj.Size)}:
    79  		case <-ctx.Done():
    80  			return context.ErrCanceled
    81  		}
    82  	}
    83  	return nil
    84  }
    85  
    86  func (gs *Storage) ReceiveBlob(br blob.Ref, source io.Reader) (blob.SizedRef, error) {
    87  	buf := &bytes.Buffer{}
    88  	size, err := io.Copy(buf, source)
    89  	if err != nil {
    90  		return blob.SizedRef{}, err
    91  	}
    92  
    93  	for tries, shouldRetry := 0, true; tries < 2 && shouldRetry; tries++ {
    94  		shouldRetry, err = gs.client.PutObject(
    95  			&googlestorage.Object{Bucket: gs.bucket, Key: br.String()},
    96  			ioutil.NopCloser(bytes.NewReader(buf.Bytes())))
    97  	}
    98  	if err != nil {
    99  		return blob.SizedRef{}, err
   100  	}
   101  
   102  	return blob.SizedRef{Ref: br, Size: uint32(size)}, nil
   103  }
   104  
   105  func (gs *Storage) StatBlobs(dest chan<- blob.SizedRef, blobs []blob.Ref) error {
   106  	var reterr error
   107  
   108  	// TODO: do a batch API call, or at least keep N of these in flight at a time. No need to do them all serially.
   109  	for _, br := range blobs {
   110  		size, _, err := gs.client.StatObject(
   111  			&googlestorage.Object{Bucket: gs.bucket, Key: br.String()})
   112  		if err == nil {
   113  			if size > constants.MaxBlobSize {
   114  				return errors.New("object too big")
   115  			}
   116  			dest <- blob.SizedRef{Ref: br, Size: uint32(size)}
   117  		} else {
   118  			reterr = err
   119  		}
   120  	}
   121  	return reterr
   122  }
   123  
   124  func (gs *Storage) Fetch(blob blob.Ref) (file io.ReadCloser, size uint32, err error) {
   125  	file, sz, err := gs.client.GetObject(&googlestorage.Object{Bucket: gs.bucket, Key: blob.String()})
   126  	if err != nil && sz > constants.MaxBlobSize {
   127  		err = errors.New("object too big")
   128  	}
   129  	return file, uint32(sz), err
   130  
   131  }
   132  
   133  func (gs *Storage) RemoveBlobs(blobs []blob.Ref) error {
   134  	var reterr error
   135  	// TODO: do a batch API call, or at least keep N of these in flight at a time. No need to do them all serially.
   136  	for _, br := range blobs {
   137  		err := gs.client.DeleteObject(&googlestorage.Object{Bucket: gs.bucket, Key: br.String()})
   138  		if err != nil {
   139  			reterr = err
   140  		}
   141  	}
   142  	return reterr
   143  }
   144  
   145  func init() {
   146  	blobserver.RegisterStorageConstructor("googlecloudstorage", blobserver.StorageConstructor(newFromConfig))
   147  }