github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/cacher/cacher.go (about)

     1  /*
     2  Copyright 2011 Google Inc.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8       http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package cacher provides various blobref fetching caching mechanisms.
    18  package cacher
    19  
    20  import (
    21  	"io"
    22  	"io/ioutil"
    23  	"log"
    24  	"os"
    25  	"path/filepath"
    26  
    27  	"camlistore.org/pkg/blob"
    28  	"camlistore.org/pkg/blobserver"
    29  	"camlistore.org/pkg/blobserver/localdisk"
    30  	"camlistore.org/pkg/osutil"
    31  	"camlistore.org/pkg/singleflight"
    32  )
    33  
    34  // NewCachingFetcher returns a CachingFetcher that fetches from
    35  // fetcher and writes to and serves from cache.
    36  func NewCachingFetcher(cache blobserver.Cache, fetcher blob.Fetcher) *CachingFetcher {
    37  	return &CachingFetcher{c: cache, sf: fetcher}
    38  }
    39  
    40  // A CachingFetcher is a blob.Fetcher and a blob.SeekFetcher.
    41  type CachingFetcher struct {
    42  	c  blobserver.Cache
    43  	sf blob.Fetcher
    44  
    45  	g singleflight.Group
    46  }
    47  
    48  func (cf *CachingFetcher) Fetch(br blob.Ref) (file io.ReadCloser, size uint32, err error) {
    49  	file, size, err = cf.c.Fetch(br)
    50  	if err == nil {
    51  		return
    52  	}
    53  	if err = cf.faultIn(br); err != nil {
    54  		return
    55  	}
    56  	return cf.c.Fetch(br)
    57  }
    58  
    59  func (cf *CachingFetcher) faultIn(br blob.Ref) error {
    60  	_, err := cf.g.Do(br.String(), func() (interface{}, error) {
    61  		sblob, _, err := cf.sf.Fetch(br)
    62  		if err != nil {
    63  			return nil, err
    64  		}
    65  		defer sblob.Close()
    66  		_, err = blobserver.Receive(cf.c, br, sblob)
    67  		return nil, err
    68  	})
    69  	return err
    70  }
    71  
    72  // A DiskCache is a blob.Fetcher that serves from a local temp
    73  // directory and is backed by a another blob.Fetcher (usually the
    74  // pkg/client HTTP client).
    75  type DiskCache struct {
    76  	*CachingFetcher
    77  
    78  	// Root is the temp directory being used to store files.
    79  	// It is available mostly for debug printing.
    80  	Root string
    81  
    82  	cleanAll bool // cleaning policy. TODO: something better.
    83  }
    84  
    85  // NewDiskCache returns a new DiskCache from a Fetcher, which
    86  // is usually the pkg/client HTTP client (which typically has much
    87  // higher latency and lower bandwidth than local disk).
    88  func NewDiskCache(fetcher blob.Fetcher) (*DiskCache, error) {
    89  	cacheDir := filepath.Join(osutil.CacheDir(), "blobs")
    90  	if !osutil.DirExists(cacheDir) {
    91  		if err := os.Mkdir(cacheDir, 0700); err != nil {
    92  			log.Printf("Warning: failed to make %s: %v; using tempdir instead", cacheDir, err)
    93  			cacheDir, err = ioutil.TempDir("", "camlicache")
    94  			if err != nil {
    95  				return nil, err
    96  			}
    97  		}
    98  	}
    99  	// TODO: max disk size, keep LRU of access, smarter cleaning, etc
   100  	// TODO: use diskpacked instead? harder to clean, though.
   101  	diskcache, err := localdisk.New(cacheDir)
   102  	if err != nil {
   103  		return nil, err
   104  	}
   105  	dc := &DiskCache{
   106  		CachingFetcher: NewCachingFetcher(diskcache, fetcher),
   107  		Root:           cacheDir,
   108  	}
   109  	return dc, nil
   110  }
   111  
   112  // Clean cleans some or all of the DiskCache.
   113  func (dc *DiskCache) Clean() {
   114  	// TODO: something between nothing and deleting everything.
   115  	if dc.cleanAll {
   116  		os.RemoveAll(dc.Root)
   117  	}
   118  }
   119  
   120  var (
   121  	_ blob.Fetcher = (*CachingFetcher)(nil)
   122  	_ blob.Fetcher = (*DiskCache)(nil)
   123  )