github.com/olivere/camlistore@v0.0.0-20140121221811-1b7ac2da0199/pkg/cacher/cacher.go (about) 1 /* 2 Copyright 2011 Google Inc. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 // Package cacher provides various blobref fetching caching mechanisms. 18 package cacher 19 20 import ( 21 "io" 22 "io/ioutil" 23 "os" 24 25 "camlistore.org/pkg/blob" 26 "camlistore.org/pkg/blobserver" 27 "camlistore.org/pkg/blobserver/localdisk" 28 "camlistore.org/pkg/singleflight" 29 "camlistore.org/pkg/types" 30 ) 31 32 // NewCachingFetcher returns a CachingFetcher that fetches from 33 // fetcher and writes to and serves from cache. 34 func NewCachingFetcher(cache blobserver.Cache, fetcher blob.StreamingFetcher) *CachingFetcher { 35 return &CachingFetcher{c: cache, sf: fetcher} 36 } 37 38 // A CachingFetcher is a blob.StreamingFetcher and a blob.SeekFetcher. 39 type CachingFetcher struct { 40 c blobserver.Cache 41 sf blob.StreamingFetcher 42 43 g singleflight.Group 44 } 45 46 func (cf *CachingFetcher) FetchStreaming(br blob.Ref) (file io.ReadCloser, size int64, err error) { 47 file, size, err = cf.c.Fetch(br) 48 if err == nil { 49 return 50 } 51 if err = cf.faultIn(br); err != nil { 52 return 53 } 54 return cf.c.Fetch(br) 55 } 56 57 func (cf *CachingFetcher) Fetch(br blob.Ref) (file types.ReadSeekCloser, size int64, err error) { 58 file, size, err = cf.c.Fetch(br) 59 if err == nil { 60 return 61 } 62 if err = cf.faultIn(br); err != nil { 63 return 64 } 65 return cf.c.Fetch(br) 66 } 67 68 func (cf *CachingFetcher) faultIn(br blob.Ref) error { 69 _, err := cf.g.Do(br.String(), func() (interface{}, error) { 70 sblob, _, err := cf.sf.FetchStreaming(br) 71 if err != nil { 72 return nil, err 73 } 74 defer sblob.Close() 75 _, err = cf.c.ReceiveBlob(br, sblob) 76 return nil, err 77 }) 78 return err 79 } 80 81 // A DiskCache is a blob.StreamingFetcher and blob.SeekFetcher 82 // that serves from a local temp directory and is backed by a another 83 // blob.StreamingFetcher (usually the pkg/client HTTP client). 84 type DiskCache struct { 85 *CachingFetcher 86 87 // Root is the temp directory being used to store files. 88 // It is available mostly for debug printing. 89 Root string 90 } 91 92 // NewDiskCache returns a new DiskCache from a StreamingFetcher, which 93 // is usually the pkg/client HTTP client (which typically has much 94 // higher latency and lower bandwidth than local disk). 95 func NewDiskCache(fetcher blob.StreamingFetcher) (*DiskCache, error) { 96 // TODO: max disk size, keep LRU of access, smarter cleaning, 97 // persistent directory per-user, etc. 98 99 cacheDir, err := ioutil.TempDir("", "camlicache") 100 if err != nil { 101 return nil, err 102 } 103 diskcache, err := localdisk.New(cacheDir) 104 if err != nil { 105 return nil, err 106 } 107 dc := &DiskCache{ 108 CachingFetcher: NewCachingFetcher(diskcache, fetcher), 109 Root: cacheDir, 110 } 111 return dc, nil 112 } 113 114 // Clean cleans some or all of the DiskCache. 115 func (dc *DiskCache) Clean() { 116 // TODO: something less aggressive? 117 os.RemoveAll(dc.Root) 118 } 119 120 var ( 121 _ blob.StreamingFetcher = (*CachingFetcher)(nil) 122 _ blob.SeekFetcher = (*CachingFetcher)(nil) 123 _ blob.StreamingFetcher = (*DiskCache)(nil) 124 _ blob.SeekFetcher = (*DiskCache)(nil) 125 )