github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/blobserver/namespace/ns.go (about)

     1  /*
     2  Copyright 2014 The Camlistore Authors
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8       http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package namespace implements the "namespace" blobserver storage type.
    18  //
    19  // A namespace storage is backed by another storage target but only
    20  // has access and visibility to a subset of the blobs which have been
    21  // uploaded through this namespace. The list of accessible blobs are
    22  // stored in the provided "inventory" sorted key/value target.
    23  package namespace
    24  
    25  import (
    26  	"bytes"
    27  	"fmt"
    28  	"io"
    29  	"log"
    30  	"os"
    31  	"strconv"
    32  
    33  	"camlistore.org/pkg/blob"
    34  	"camlistore.org/pkg/blobserver"
    35  	"camlistore.org/pkg/context"
    36  	"camlistore.org/pkg/jsonconfig"
    37  	"camlistore.org/pkg/sorted"
    38  	"camlistore.org/pkg/strutil"
    39  )
    40  
    41  type nsto struct {
    42  	inventory sorted.KeyValue
    43  	master    blobserver.Storage
    44  }
    45  
    46  func init() {
    47  	blobserver.RegisterStorageConstructor("namespace", blobserver.StorageConstructor(newFromConfig))
    48  }
    49  
    50  func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
    51  	sto := &nsto{}
    52  	invConf := config.RequiredObject("inventory")
    53  	masterName := config.RequiredString("storage")
    54  	if err := config.Validate(); err != nil {
    55  		return nil, err
    56  	}
    57  	sto.inventory, err = sorted.NewKeyValue(invConf)
    58  	if err != nil {
    59  		return nil, fmt.Errorf("Invalid 'inventory' configuration: %v", err)
    60  	}
    61  	sto.master, err = ld.GetStorage(masterName)
    62  	if err != nil {
    63  		return nil, fmt.Errorf("Invalid 'storage' configuration: %v", err)
    64  	}
    65  	return sto, nil
    66  }
    67  
    68  func (ns *nsto) EnumerateBlobs(ctx *context.Context, dest chan<- blob.SizedRef, after string, limit int) error {
    69  	defer close(dest)
    70  	done := ctx.Done()
    71  
    72  	it := ns.inventory.Find(after, "")
    73  	first := true
    74  	for limit > 0 && it.Next() {
    75  		if first {
    76  			first = false
    77  			if after != "" && it.Key() == after {
    78  				continue
    79  			}
    80  		}
    81  		br, ok := blob.ParseBytes(it.KeyBytes())
    82  		size, err := strutil.ParseUintBytes(it.ValueBytes(), 10, 32)
    83  		if !ok || err != nil {
    84  			log.Printf("Bogus namespace key %q / value %q", it.Key(), it.Value())
    85  			continue
    86  		}
    87  		select {
    88  		case dest <- blob.SizedRef{br, uint32(size)}:
    89  		case <-done:
    90  			return context.ErrCanceled
    91  		}
    92  		limit--
    93  	}
    94  	if err := it.Close(); err != nil {
    95  		return err
    96  	}
    97  	return nil
    98  }
    99  
   100  func (ns *nsto) Fetch(br blob.Ref) (rc io.ReadCloser, size uint32, err error) {
   101  	invSizeStr, err := ns.inventory.Get(br.String())
   102  	if err == sorted.ErrNotFound {
   103  		err = os.ErrNotExist
   104  		return
   105  	}
   106  	if err != nil {
   107  		return
   108  	}
   109  	invSize, err := strconv.ParseUint(invSizeStr, 10, 32)
   110  	if err != nil {
   111  		return
   112  	}
   113  	rc, size, err = ns.master.Fetch(br)
   114  	if err != nil {
   115  		return
   116  	}
   117  	if size != uint32(invSize) {
   118  		log.Printf("namespace: on blob %v, unexpected inventory size %d for master size %d", br, invSize, size)
   119  		return nil, 0, os.ErrNotExist
   120  	}
   121  	return rc, size, nil
   122  }
   123  
   124  func (ns *nsto) ReceiveBlob(br blob.Ref, src io.Reader) (sb blob.SizedRef, err error) {
   125  	var buf bytes.Buffer
   126  	size, err := io.Copy(&buf, src)
   127  	if err != nil {
   128  		return
   129  	}
   130  
   131  	// Check if a duplicate blob, already uploaded previously.
   132  	if _, ierr := ns.inventory.Get(br.String()); ierr == nil {
   133  		return blob.SizedRef{br, uint32(size)}, nil
   134  	}
   135  
   136  	sb, err = ns.master.ReceiveBlob(br, &buf)
   137  	if err != nil {
   138  		return
   139  	}
   140  
   141  	err = ns.inventory.Set(br.String(), strconv.Itoa(int(size)))
   142  	return
   143  }
   144  
   145  func (ns *nsto) RemoveBlobs(blobs []blob.Ref) error {
   146  	for _, br := range blobs {
   147  		if err := ns.inventory.Delete(br.String()); err != nil {
   148  			return err
   149  		}
   150  	}
   151  	return nil
   152  }
   153  
   154  func (ns *nsto) StatBlobs(dest chan<- blob.SizedRef, blobs []blob.Ref) error {
   155  	for _, br := range blobs {
   156  		invSizeStr, err := ns.inventory.Get(br.String())
   157  		if err == sorted.ErrNotFound {
   158  			continue
   159  		}
   160  		if err != nil {
   161  			return err
   162  		}
   163  		invSize, err := strconv.ParseUint(invSizeStr, 10, 32)
   164  		if err != nil {
   165  			log.Printf("Bogus namespace key %q / value %q", br.String(), invSizeStr)
   166  		}
   167  		dest <- blob.SizedRef{br, uint32(invSize)}
   168  	}
   169  	return nil
   170  }