github.com/olivere/camlistore@v0.0.0-20140121221811-1b7ac2da0199/pkg/cacher/cacher.go (about)

     1  /*
     2  Copyright 2011 Google Inc.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8       http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package cacher provides various blobref fetching caching mechanisms.
    18  package cacher
    19  
    20  import (
    21  	"io"
    22  	"io/ioutil"
    23  	"os"
    24  
    25  	"camlistore.org/pkg/blob"
    26  	"camlistore.org/pkg/blobserver"
    27  	"camlistore.org/pkg/blobserver/localdisk"
    28  	"camlistore.org/pkg/singleflight"
    29  	"camlistore.org/pkg/types"
    30  )
    31  
    32  // NewCachingFetcher returns a CachingFetcher that fetches from
    33  // fetcher and writes to and serves from cache.
    34  func NewCachingFetcher(cache blobserver.Cache, fetcher blob.StreamingFetcher) *CachingFetcher {
    35  	return &CachingFetcher{c: cache, sf: fetcher}
    36  }
    37  
    38  // A CachingFetcher is a blob.StreamingFetcher and a blob.SeekFetcher.
    39  type CachingFetcher struct {
    40  	c  blobserver.Cache
    41  	sf blob.StreamingFetcher
    42  
    43  	g singleflight.Group
    44  }
    45  
    46  func (cf *CachingFetcher) FetchStreaming(br blob.Ref) (file io.ReadCloser, size int64, err error) {
    47  	file, size, err = cf.c.Fetch(br)
    48  	if err == nil {
    49  		return
    50  	}
    51  	if err = cf.faultIn(br); err != nil {
    52  		return
    53  	}
    54  	return cf.c.Fetch(br)
    55  }
    56  
    57  func (cf *CachingFetcher) Fetch(br blob.Ref) (file types.ReadSeekCloser, size int64, err error) {
    58  	file, size, err = cf.c.Fetch(br)
    59  	if err == nil {
    60  		return
    61  	}
    62  	if err = cf.faultIn(br); err != nil {
    63  		return
    64  	}
    65  	return cf.c.Fetch(br)
    66  }
    67  
    68  func (cf *CachingFetcher) faultIn(br blob.Ref) error {
    69  	_, err := cf.g.Do(br.String(), func() (interface{}, error) {
    70  		sblob, _, err := cf.sf.FetchStreaming(br)
    71  		if err != nil {
    72  			return nil, err
    73  		}
    74  		defer sblob.Close()
    75  		_, err = cf.c.ReceiveBlob(br, sblob)
    76  		return nil, err
    77  	})
    78  	return err
    79  }
    80  
    81  // A DiskCache is a blob.StreamingFetcher and blob.SeekFetcher
    82  // that serves from a local temp directory and is backed by a another
    83  // blob.StreamingFetcher (usually the pkg/client HTTP client).
    84  type DiskCache struct {
    85  	*CachingFetcher
    86  
    87  	// Root is the temp directory being used to store files.
    88  	// It is available mostly for debug printing.
    89  	Root string
    90  }
    91  
    92  // NewDiskCache returns a new DiskCache from a StreamingFetcher, which
    93  // is usually the pkg/client HTTP client (which typically has much
    94  // higher latency and lower bandwidth than local disk).
    95  func NewDiskCache(fetcher blob.StreamingFetcher) (*DiskCache, error) {
    96  	// TODO: max disk size, keep LRU of access, smarter cleaning,
    97  	// persistent directory per-user, etc.
    98  
    99  	cacheDir, err := ioutil.TempDir("", "camlicache")
   100  	if err != nil {
   101  		return nil, err
   102  	}
   103  	diskcache, err := localdisk.New(cacheDir)
   104  	if err != nil {
   105  		return nil, err
   106  	}
   107  	dc := &DiskCache{
   108  		CachingFetcher: NewCachingFetcher(diskcache, fetcher),
   109  		Root:           cacheDir,
   110  	}
   111  	return dc, nil
   112  }
   113  
   114  // Clean cleans some or all of the DiskCache.
   115  func (dc *DiskCache) Clean() {
   116  	// TODO: something less aggressive?
   117  	os.RemoveAll(dc.Root)
   118  }
   119  
   120  var (
   121  	_ blob.StreamingFetcher = (*CachingFetcher)(nil)
   122  	_ blob.SeekFetcher      = (*CachingFetcher)(nil)
   123  	_ blob.StreamingFetcher = (*DiskCache)(nil)
   124  	_ blob.SeekFetcher      = (*DiskCache)(nil)
   125  )