github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/blobserver/namespace/ns.go (about) 1 /* 2 Copyright 2014 The Camlistore Authors 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 // Package namespace implements the "namespace" blobserver storage type. 18 // 19 // A namespace storage is backed by another storage target but only 20 // has access and visibility to a subset of the blobs which have been 21 // uploaded through this namespace. The list of accessible blobs are 22 // stored in the provided "inventory" sorted key/value target. 23 package namespace 24 25 import ( 26 "bytes" 27 "fmt" 28 "io" 29 "log" 30 "os" 31 "strconv" 32 33 "camlistore.org/pkg/blob" 34 "camlistore.org/pkg/blobserver" 35 "camlistore.org/pkg/context" 36 "camlistore.org/pkg/jsonconfig" 37 "camlistore.org/pkg/sorted" 38 "camlistore.org/pkg/strutil" 39 ) 40 41 type nsto struct { 42 inventory sorted.KeyValue 43 master blobserver.Storage 44 } 45 46 func init() { 47 blobserver.RegisterStorageConstructor("namespace", blobserver.StorageConstructor(newFromConfig)) 48 } 49 50 func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) { 51 sto := &nsto{} 52 invConf := config.RequiredObject("inventory") 53 masterName := config.RequiredString("storage") 54 if err := config.Validate(); err != nil { 55 return nil, err 56 } 57 sto.inventory, err = sorted.NewKeyValue(invConf) 58 if err != nil { 59 return nil, fmt.Errorf("Invalid 'inventory' configuration: %v", err) 60 } 61 sto.master, err = ld.GetStorage(masterName) 62 if err != nil { 63 return nil, fmt.Errorf("Invalid 'storage' configuration: %v", err) 64 } 65 return sto, nil 66 } 67 68 func (ns *nsto) EnumerateBlobs(ctx *context.Context, dest chan<- blob.SizedRef, after string, limit int) error { 69 defer close(dest) 70 done := ctx.Done() 71 72 it := ns.inventory.Find(after, "") 73 first := true 74 for limit > 0 && it.Next() { 75 if first { 76 first = false 77 if after != "" && it.Key() == after { 78 continue 79 } 80 } 81 br, ok := blob.ParseBytes(it.KeyBytes()) 82 size, err := strutil.ParseUintBytes(it.ValueBytes(), 10, 32) 83 if !ok || err != nil { 84 log.Printf("Bogus namespace key %q / value %q", it.Key(), it.Value()) 85 continue 86 } 87 select { 88 case dest <- blob.SizedRef{br, uint32(size)}: 89 case <-done: 90 return context.ErrCanceled 91 } 92 limit-- 93 } 94 if err := it.Close(); err != nil { 95 return err 96 } 97 return nil 98 } 99 100 func (ns *nsto) Fetch(br blob.Ref) (rc io.ReadCloser, size uint32, err error) { 101 invSizeStr, err := ns.inventory.Get(br.String()) 102 if err == sorted.ErrNotFound { 103 err = os.ErrNotExist 104 return 105 } 106 if err != nil { 107 return 108 } 109 invSize, err := strconv.ParseUint(invSizeStr, 10, 32) 110 if err != nil { 111 return 112 } 113 rc, size, err = ns.master.Fetch(br) 114 if err != nil { 115 return 116 } 117 if size != uint32(invSize) { 118 log.Printf("namespace: on blob %v, unexpected inventory size %d for master size %d", br, invSize, size) 119 return nil, 0, os.ErrNotExist 120 } 121 return rc, size, nil 122 } 123 124 func (ns *nsto) ReceiveBlob(br blob.Ref, src io.Reader) (sb blob.SizedRef, err error) { 125 var buf bytes.Buffer 126 size, err := io.Copy(&buf, src) 127 if err != nil { 128 return 129 } 130 131 // Check if a duplicate blob, already uploaded previously. 132 if _, ierr := ns.inventory.Get(br.String()); ierr == nil { 133 return blob.SizedRef{br, uint32(size)}, nil 134 } 135 136 sb, err = ns.master.ReceiveBlob(br, &buf) 137 if err != nil { 138 return 139 } 140 141 err = ns.inventory.Set(br.String(), strconv.Itoa(int(size))) 142 return 143 } 144 145 func (ns *nsto) RemoveBlobs(blobs []blob.Ref) error { 146 for _, br := range blobs { 147 if err := ns.inventory.Delete(br.String()); err != nil { 148 return err 149 } 150 } 151 return nil 152 } 153 154 func (ns *nsto) StatBlobs(dest chan<- blob.SizedRef, blobs []blob.Ref) error { 155 for _, br := range blobs { 156 invSizeStr, err := ns.inventory.Get(br.String()) 157 if err == sorted.ErrNotFound { 158 continue 159 } 160 if err != nil { 161 return err 162 } 163 invSize, err := strconv.ParseUint(invSizeStr, 10, 32) 164 if err != nil { 165 log.Printf("Bogus namespace key %q / value %q", br.String(), invSizeStr) 166 } 167 dest <- blob.SizedRef{br, uint32(invSize)} 168 } 169 return nil 170 }