github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/blobserver/remote/remote.go (about)

     1  /*
     2  Copyright 2011 Google Inc.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8       http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  /*
    18  Package remote registers the "remote" blobserver storage type, storing
    19  and fetching blobs from a remote Camlistore server, speaking the HTTP
    20  protocol.
    21  
    22  Example low-level config:
    23  
    24       "/peer/": {
    25           "handler": "storage-remote",
    26           "handlerArgs": {
    27               "url": "http://10.0.0.17/base",
    28               "auth": "userpass:user:pass",
    29               "skipStartupCheck": false
    30            }
    31       },
    32  
    33  */
    34  package remote
    35  
    36  import (
    37  	"io"
    38  	"log"
    39  	"os"
    40  
    41  	"camlistore.org/pkg/blob"
    42  	"camlistore.org/pkg/blobserver"
    43  	"camlistore.org/pkg/client"
    44  	"camlistore.org/pkg/context"
    45  	"camlistore.org/pkg/jsonconfig"
    46  )
    47  
    48  // remoteStorage is a blobserver.Storage proxy for a remote camlistore
    49  // blobserver.
    50  type remoteStorage struct {
    51  	client *client.Client
    52  }
    53  
    54  var _ = blobserver.Storage((*remoteStorage)(nil))
    55  
    56  // NewFromClient returns a new Storage implementation using the
    57  // provided Camlistore client.
    58  func NewFromClient(c *client.Client) blobserver.Storage {
    59  	return &remoteStorage{client: c}
    60  }
    61  
    62  func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
    63  	url := config.RequiredString("url")
    64  	auth := config.RequiredString("auth")
    65  	skipStartupCheck := config.OptionalBool("skipStartupCheck", false)
    66  	if err := config.Validate(); err != nil {
    67  		return nil, err
    68  	}
    69  
    70  	client := client.New(url)
    71  	if err = client.SetupAuthFromString(auth); err != nil {
    72  		return nil, err
    73  	}
    74  	client.SetLogger(log.New(os.Stderr, "remote", log.LstdFlags))
    75  	sto := &remoteStorage{
    76  		client: client,
    77  	}
    78  	if !skipStartupCheck {
    79  		// Do a quick dummy operation to check that our credentials are
    80  		// correct.
    81  		// TODO(bradfitz,mpl): skip this operation smartly if it turns out this is annoying/slow for whatever reason.
    82  		c := make(chan blob.SizedRef, 1)
    83  		err = sto.EnumerateBlobs(context.TODO(), c, "", 1)
    84  		if err != nil {
    85  			return nil, err
    86  		}
    87  	}
    88  	return sto, nil
    89  }
    90  
    91  func (sto *remoteStorage) RemoveBlobs(blobs []blob.Ref) error {
    92  	return sto.client.RemoveBlobs(blobs)
    93  }
    94  
    95  func (sto *remoteStorage) StatBlobs(dest chan<- blob.SizedRef, blobs []blob.Ref) error {
    96  	// TODO: cache the stat response's uploadUrl to save a future
    97  	// stat later?  otherwise clients will just Stat + Upload, but
    98  	// Upload will also Stat.  should be smart and make sure we
    99  	// avoid ReceiveBlob's Stat whenever it would be redundant.
   100  	return sto.client.StatBlobs(dest, blobs)
   101  }
   102  
   103  func (sto *remoteStorage) ReceiveBlob(blob blob.Ref, source io.Reader) (outsb blob.SizedRef, outerr error) {
   104  	h := &client.UploadHandle{
   105  		BlobRef:  blob,
   106  		Size:     0, // size isn't known; 0 is fine, but TODO: ask source if it knows its size
   107  		Contents: source,
   108  	}
   109  	pr, err := sto.client.Upload(h)
   110  	if err != nil {
   111  		outerr = err
   112  		return
   113  	}
   114  	return pr.SizedBlobRef(), nil
   115  }
   116  
   117  func (sto *remoteStorage) Fetch(b blob.Ref) (file io.ReadCloser, size uint32, err error) {
   118  	return sto.client.Fetch(b)
   119  }
   120  
   121  func (sto *remoteStorage) MaxEnumerate() int { return 1000 }
   122  
   123  func (sto *remoteStorage) EnumerateBlobs(ctx *context.Context, dest chan<- blob.SizedRef, after string, limit int) error {
   124  	return sto.client.EnumerateBlobsOpts(ctx, dest, client.EnumerateOpts{
   125  		After: after,
   126  		Limit: limit,
   127  	})
   128  }
   129  
   130  func init() {
   131  	blobserver.RegisterStorageConstructor("remote", blobserver.StorageConstructor(newFromConfig))
   132  }