github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/blobserver/interface.go (about) 1 /* 2 Copyright 2011 Google Inc. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package blobserver 18 19 import ( 20 "errors" 21 "io" 22 "net/http" 23 "os" 24 "time" 25 26 "camlistore.org/pkg/blob" 27 "camlistore.org/pkg/constants" 28 "camlistore.org/pkg/context" 29 ) 30 31 // MaxBlobSize is the size of a single blob in Camlistore. 32 const MaxBlobSize = constants.MaxBlobSize 33 34 var ErrCorruptBlob = errors.New("corrupt blob; digest doesn't match") 35 36 // ErrNotImplemented should be returned in methods where the function is not implemented 37 var ErrNotImplemented = errors.New("not implemented") 38 39 // BlobReceiver is the interface for receiving 40 type BlobReceiver interface { 41 // ReceiveBlob accepts a newly uploaded blob and writes it to 42 // permanent storage. 43 // 44 // Implementations of BlobReceiver downstream of the HTTP 45 // server can trust that the source isn't larger than 46 // MaxBlobSize and that its digest matches the provided blob 47 // ref. (If not, the read of the source will fail before EOF) 48 // 49 // To ensure those guarantees, callers of ReceiveBlob should 50 // not call ReceiveBlob directly but instead use either 51 // blobserver.Receive or blobserver.ReceiveString, which also 52 // take care of notifying the BlobReceiver's "BlobHub" 53 // notification bus for observers. 54 ReceiveBlob(br blob.Ref, source io.Reader) (blob.SizedRef, error) 55 } 56 57 type BlobStatter interface { 58 // Stat checks for the existence of blobs, writing their sizes 59 // (if found back to the dest channel), and returning an error 60 // or nil. Stat() should NOT close the channel. 61 // TODO(bradfitz): redefine this to close the channel? Or document 62 // better what the synchronization rules are. 63 StatBlobs(dest chan<- blob.SizedRef, blobs []blob.Ref) error 64 } 65 66 func StatBlob(bs BlobStatter, br blob.Ref) (sb blob.SizedRef, err error) { 67 c := make(chan blob.SizedRef, 1) 68 err = bs.StatBlobs(c, []blob.Ref{br}) 69 if err != nil { 70 return 71 } 72 select { 73 case sb = <-c: 74 default: 75 err = os.ErrNotExist 76 } 77 return 78 } 79 80 type StatReceiver interface { 81 BlobReceiver 82 BlobStatter 83 } 84 85 type BlobEnumerator interface { 86 // EnumerateBobs sends at most limit SizedBlobRef into dest, 87 // sorted, as long as they are lexigraphically greater than 88 // after (if provided). 89 // limit will be supplied and sanity checked by caller. 90 // EnumerateBlobs must close the channel. (even if limit 91 // was hit and more blobs remain, or an error is returned, or 92 // the ctx is canceled) 93 EnumerateBlobs(ctx *context.Context, 94 dest chan<- blob.SizedRef, 95 after string, 96 limit int) error 97 } 98 99 type BlobStreamer interface { 100 // StreamBlobs sends blobs to dest in unspecified order. It is 101 // expected that a blobstorage implementing BlobStreamer will 102 // send blobs to dest in the most efficient order 103 // possible. StreamBlobs will stop sending blobs to dest and 104 // return an opaque continuation token (in the string return 105 // parameter) when the total size of the blobs it has sent 106 // equals or exceeds limit. A succeeding call to StreamBlobs 107 // should pass the string returned from the previous call in 108 // contToken, or an empty string if the caller wishes to 109 // receive blobs from "the start". StreamBlobs must 110 // unconditionally close dest before returning, and it must 111 // return if ctx.Done() becomes readable. 112 StreamBlobs(ctx *context.Context, dest chan<- *blob.Blob, contToken string, limitBytes int64) (nextContinueToken string, err error) 113 } 114 115 // Cache is the minimal interface expected of a blob cache. 116 type Cache interface { 117 blob.Fetcher 118 BlobReceiver 119 BlobStatter 120 } 121 122 type BlobReceiveConfiger interface { 123 BlobReceiver 124 Configer 125 } 126 127 type Config struct { 128 Writable bool 129 Readable bool 130 Deletable bool 131 CanLongPoll bool 132 133 // the "http://host:port" and optional path (but without trailing slash) to have "/camli/*" appended 134 URLBase string 135 HandlerFinder FindHandlerByTyper 136 } 137 138 type BlobRemover interface { 139 // RemoveBlobs removes 0 or more blobs. Removal of 140 // non-existent items isn't an error. Returns failure if any 141 // items existed but failed to be deleted. 142 // ErrNotImplemented may be returned for storage types not implementing removal. 143 RemoveBlobs(blobs []blob.Ref) error 144 } 145 146 // Storage is the interface that must be implemented by a blobserver 147 // storage type. (e.g. localdisk, s3, encrypt, shard, replica, remote) 148 type Storage interface { 149 blob.Fetcher 150 BlobReceiver 151 BlobStatter 152 BlobEnumerator 153 BlobRemover 154 } 155 156 type FetcherEnumerator interface { 157 blob.Fetcher 158 BlobEnumerator 159 } 160 161 // StorageHandler is a storage implementation that also exports an HTTP 162 // status page. 163 type StorageHandler interface { 164 Storage 165 http.Handler 166 } 167 168 // Optional interface for storage implementations which can be asked 169 // to shut down cleanly. Regardless, all implementations should 170 // be able to survive crashes without data loss. 171 type ShutdownStorage interface { 172 Storage 173 io.Closer 174 } 175 176 // A GenerationNotSupportedError explains why a Storage 177 // value implemented the Generationer interface but failed due 178 // to a wrapped Storage value not implementing the interface. 179 type GenerationNotSupportedError string 180 181 func (s GenerationNotSupportedError) Error() string { return string(s) } 182 183 /* 184 The optional Generationer interface is an optimization and paranoia 185 facility for clients which can be implemented by Storage 186 implementations. 187 188 If the client sees the same random string in multiple upload sessions, 189 it assumes that the blobserver still has all the same blobs, and also 190 it's the same server. This mechanism is not fundamental to 191 Camlistore's operation: the client could also check each blob before 192 uploading, or enumerate all blobs from the server too. This is purely 193 an optimization so clients can mix this value into their "is this file 194 uploaded?" local cache keys. 195 */ 196 type Generationer interface { 197 // Generation returns a Storage's initialization time and 198 // and unique random string (or UUID). Implementations 199 // should call ResetStorageGeneration on demand if no 200 // information is known. 201 // The error will be of type GenerationNotSupportedError if an underlying 202 // storage target doesn't support the Generationer interface. 203 StorageGeneration() (initTime time.Time, random string, err error) 204 205 // ResetGeneration deletes the information returned by Generation 206 // and re-generates it. 207 ResetStorageGeneration() error 208 } 209 210 type Configer interface { 211 Config() *Config 212 } 213 214 type StorageConfiger interface { 215 Storage 216 Configer 217 } 218 219 // MaxEnumerateConfig is an optional interface implemented by Storage 220 // interfaces to advertise their max value for how many items can 221 // be enumerated at once. 222 type MaxEnumerateConfig interface { 223 Storage 224 225 // MaxEnumerate returns the max that this storage interface is 226 // capable of enumerating at once. 227 MaxEnumerate() int 228 }