github.com/SaurabhDubey-Groww/go-cloud@v0.0.0-20221124105541-b26c29285fd8/blob/memblob/memblob.go (about) 1 // Copyright 2018 The Go Cloud Development Kit Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // https://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // Package memblob provides an in-memory blob implementation. 16 // Use OpenBucket to construct a *blob.Bucket. 17 // 18 // # URLs 19 // 20 // For blob.OpenBucket memblob registers for the scheme "mem". 21 // To customize the URL opener, or for more details on the URL format, 22 // see URLOpener. 23 // See https://gocloud.dev/concepts/urls/ for background information. 24 // 25 // # As 26 // 27 // memblob does not support any types for As. 28 package memblob // import "gocloud.dev/blob/memblob" 29 30 import ( 31 "bytes" 32 "context" 33 "crypto/md5" 34 "errors" 35 "fmt" 36 "hash" 37 "io" 38 "net/url" 39 "sort" 40 "strings" 41 "sync" 42 "time" 43 44 "gocloud.dev/blob" 45 "gocloud.dev/blob/driver" 46 "gocloud.dev/gcerrors" 47 ) 48 49 const defaultPageSize = 1000 50 51 var ( 52 errNotFound = errors.New("blob not found") 53 errNotImplemented = errors.New("not implemented") 54 ) 55 56 func init() { 57 blob.DefaultURLMux().RegisterBucket(Scheme, &URLOpener{}) 58 } 59 60 // Scheme is the URL scheme memblob registers its URLOpener under on 61 // blob.DefaultMux. 62 const Scheme = "mem" 63 64 // URLOpener opens URLs like "mem://". 65 // 66 // No query parameters are supported. 67 type URLOpener struct{} 68 69 // OpenBucketURL opens a blob.Bucket based on u. 70 func (*URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket, error) { 71 for param := range u.Query() { 72 return nil, fmt.Errorf("open bucket %v: invalid query parameter %q", u, param) 73 } 74 return OpenBucket(nil), nil 75 } 76 77 // Options sets options for constructing a *blob.Bucket backed by memory. 78 type Options struct{} 79 80 type blobEntry struct { 81 Content []byte 82 Attributes *driver.Attributes 83 } 84 85 type bucket struct { 86 mu sync.Mutex 87 blobs map[string]*blobEntry 88 } 89 90 // openBucket creates a driver.Bucket backed by memory. 91 func openBucket(_ *Options) driver.Bucket { 92 return &bucket{ 93 blobs: map[string]*blobEntry{}, 94 } 95 } 96 97 // OpenBucket creates a *blob.Bucket backed by memory. 98 func OpenBucket(opts *Options) *blob.Bucket { 99 return blob.NewBucket(openBucket(opts)) 100 } 101 102 func (b *bucket) Close() error { 103 return nil 104 } 105 106 func (b *bucket) ErrorCode(err error) gcerrors.ErrorCode { 107 switch err { 108 case errNotFound: 109 return gcerrors.NotFound 110 case errNotImplemented: 111 return gcerrors.Unimplemented 112 default: 113 return gcerrors.Unknown 114 } 115 } 116 117 // ListPaged implements driver.ListPaged. 118 // The implementation largely mirrors the one in fileblob. 119 func (b *bucket) ListPaged(ctx context.Context, opts *driver.ListOptions) (*driver.ListPage, error) { 120 b.mu.Lock() 121 defer b.mu.Unlock() 122 123 // pageToken is a returned NextPageToken, set below; it's the last key of the 124 // previous page. 125 var pageToken string 126 if len(opts.PageToken) > 0 { 127 pageToken = string(opts.PageToken) 128 } 129 pageSize := opts.PageSize 130 if pageSize == 0 { 131 pageSize = defaultPageSize 132 } 133 134 var keys []string 135 for key := range b.blobs { 136 keys = append(keys, key) 137 } 138 sort.Strings(keys) 139 140 // If opts.Delimiter != "", lastPrefix contains the last "directory" key we 141 // added. It is used to avoid adding it again; all files in this "directory" 142 // are collapsed to the single directory entry. 143 var lastPrefix string 144 var result driver.ListPage 145 for _, key := range keys { 146 // Skip keys that don't match the Prefix. 147 if !strings.HasPrefix(key, opts.Prefix) { 148 continue 149 } 150 151 entry := b.blobs[key] 152 obj := &driver.ListObject{ 153 Key: key, 154 ModTime: entry.Attributes.ModTime, 155 Size: entry.Attributes.Size, 156 MD5: entry.Attributes.MD5, 157 } 158 159 // If using Delimiter, collapse "directories". 160 if opts.Delimiter != "" { 161 // Strip the prefix, which may contain Delimiter. 162 keyWithoutPrefix := key[len(opts.Prefix):] 163 // See if the key still contains Delimiter. 164 // If no, it's a file and we just include it. 165 // If yes, it's a file in a "sub-directory" and we want to collapse 166 // all files in that "sub-directory" into a single "directory" result. 167 if idx := strings.Index(keyWithoutPrefix, opts.Delimiter); idx != -1 { 168 prefix := opts.Prefix + keyWithoutPrefix[0:idx+len(opts.Delimiter)] 169 // We've already included this "directory"; don't add it. 170 if prefix == lastPrefix { 171 continue 172 } 173 // Update the object to be a "directory". 174 obj = &driver.ListObject{ 175 Key: prefix, 176 IsDir: true, 177 } 178 lastPrefix = prefix 179 } 180 } 181 182 // If there's a pageToken, skip anything before it. 183 if pageToken != "" && obj.Key <= pageToken { 184 continue 185 } 186 187 // If we've already got a full page of results, set NextPageToken and return. 188 if len(result.Objects) == pageSize { 189 result.NextPageToken = []byte(result.Objects[pageSize-1].Key) 190 return &result, nil 191 } 192 result.Objects = append(result.Objects, obj) 193 } 194 return &result, nil 195 } 196 197 // As implements driver.As. 198 func (b *bucket) As(i interface{}) bool { return false } 199 200 // As implements driver.ErrorAs. 201 func (b *bucket) ErrorAs(err error, i interface{}) bool { return false } 202 203 // Attributes implements driver.Attributes. 204 func (b *bucket) Attributes(ctx context.Context, key string) (*driver.Attributes, error) { 205 b.mu.Lock() 206 defer b.mu.Unlock() 207 208 entry, found := b.blobs[key] 209 if !found { 210 return nil, errNotFound 211 } 212 return entry.Attributes, nil 213 } 214 215 // NewRangeReader implements driver.NewRangeReader. 216 func (b *bucket) NewRangeReader(ctx context.Context, key string, offset, length int64, opts *driver.ReaderOptions) (driver.Reader, error) { 217 b.mu.Lock() 218 defer b.mu.Unlock() 219 220 entry, found := b.blobs[key] 221 if !found { 222 return nil, errNotFound 223 } 224 225 if opts.BeforeRead != nil { 226 if err := opts.BeforeRead(func(interface{}) bool { return false }); err != nil { 227 return nil, err 228 } 229 } 230 r := bytes.NewReader(entry.Content) 231 if offset > 0 { 232 if _, err := r.Seek(offset, io.SeekStart); err != nil { 233 return nil, err 234 } 235 } 236 var ior io.Reader = r 237 if length >= 0 { 238 ior = io.LimitReader(r, length) 239 } 240 return &reader{ 241 r: ior, 242 attrs: driver.ReaderAttributes{ 243 ContentType: entry.Attributes.ContentType, 244 ModTime: entry.Attributes.ModTime, 245 Size: entry.Attributes.Size, 246 }, 247 }, nil 248 } 249 250 type reader struct { 251 r io.Reader 252 attrs driver.ReaderAttributes 253 } 254 255 func (r *reader) Read(p []byte) (int, error) { 256 return r.r.Read(p) 257 } 258 259 func (r *reader) Close() error { 260 return nil 261 } 262 263 func (r *reader) Attributes() *driver.ReaderAttributes { 264 return &r.attrs 265 } 266 267 func (r *reader) As(i interface{}) bool { return false } 268 269 // NewTypedWriter implements driver.NewTypedWriter. 270 func (b *bucket) NewTypedWriter(ctx context.Context, key string, contentType string, opts *driver.WriterOptions) (driver.Writer, error) { 271 if key == "" { 272 return nil, errors.New("invalid key (empty string)") 273 } 274 b.mu.Lock() 275 defer b.mu.Unlock() 276 277 if opts.BeforeWrite != nil { 278 if err := opts.BeforeWrite(func(interface{}) bool { return false }); err != nil { 279 return nil, err 280 } 281 } 282 md := map[string]string{} 283 for k, v := range opts.Metadata { 284 md[k] = v 285 } 286 return &writer{ 287 ctx: ctx, 288 b: b, 289 key: key, 290 contentType: contentType, 291 metadata: md, 292 opts: opts, 293 md5hash: md5.New(), 294 }, nil 295 } 296 297 type writer struct { 298 ctx context.Context 299 b *bucket 300 key string 301 contentType string 302 metadata map[string]string 303 opts *driver.WriterOptions 304 buf bytes.Buffer 305 // We compute the MD5 hash so that we can store it with the file attributes, 306 // not for verification. 307 md5hash hash.Hash 308 } 309 310 func (w *writer) Write(p []byte) (n int, err error) { 311 if _, err := w.md5hash.Write(p); err != nil { 312 return 0, err 313 } 314 return w.buf.Write(p) 315 } 316 317 func (w *writer) Close() error { 318 // Check if the write was cancelled. 319 if err := w.ctx.Err(); err != nil { 320 return err 321 } 322 323 md5sum := w.md5hash.Sum(nil) 324 content := w.buf.Bytes() 325 now := time.Now() 326 entry := &blobEntry{ 327 Content: content, 328 Attributes: &driver.Attributes{ 329 CacheControl: w.opts.CacheControl, 330 ContentDisposition: w.opts.ContentDisposition, 331 ContentEncoding: w.opts.ContentEncoding, 332 ContentLanguage: w.opts.ContentLanguage, 333 ContentType: w.contentType, 334 Metadata: w.metadata, 335 Size: int64(len(content)), 336 CreateTime: now, 337 ModTime: now, 338 MD5: md5sum, 339 ETag: fmt.Sprintf("\"%x-%x\"", now.UnixNano(), len(content)), 340 }, 341 } 342 w.b.mu.Lock() 343 defer w.b.mu.Unlock() 344 if prev := w.b.blobs[w.key]; prev != nil { 345 entry.Attributes.CreateTime = prev.Attributes.CreateTime 346 } 347 w.b.blobs[w.key] = entry 348 return nil 349 } 350 351 // Copy implements driver.Copy. 352 func (b *bucket) Copy(ctx context.Context, dstKey, srcKey string, opts *driver.CopyOptions) error { 353 b.mu.Lock() 354 defer b.mu.Unlock() 355 356 if opts.BeforeCopy != nil { 357 if err := opts.BeforeCopy(func(interface{}) bool { return false }); err != nil { 358 return err 359 } 360 } 361 v := b.blobs[srcKey] 362 if v == nil { 363 return errNotFound 364 } 365 b.blobs[dstKey] = v 366 return nil 367 } 368 369 // Delete implements driver.Delete. 370 func (b *bucket) Delete(ctx context.Context, key string) error { 371 b.mu.Lock() 372 defer b.mu.Unlock() 373 374 if b.blobs[key] == nil { 375 return errNotFound 376 } 377 delete(b.blobs, key) 378 return nil 379 } 380 381 func (b *bucket) SignedURL(ctx context.Context, key string, opts *driver.SignedURLOptions) (string, error) { 382 return "", errNotImplemented 383 }