github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/nbs/fs_table_cache.go (about)

     1  // Copyright 2019 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  //
    15  // This file incorporates work covered by the following copyright and
    16  // permission notice:
    17  //
    18  // Copyright 2017 Attic Labs, Inc. All rights reserved.
    19  // Licensed under the Apache License, version 2.0:
    20  // http://www.apache.org/licenses/LICENSE-2.0
    21  
    22  package nbs
    23  
    24  import (
    25  	"errors"
    26  	"io"
    27  	"os"
    28  	"path/filepath"
    29  	"strings"
    30  	"sync"
    31  
    32  	"github.com/dolthub/dolt/go/store/atomicerr"
    33  	"github.com/dolthub/dolt/go/store/util/sizecache"
    34  	"github.com/dolthub/dolt/go/store/util/tempfiles"
    35  )
    36  
    37  type tableCache interface {
    38  	checkout(h addr) (io.ReaderAt, error)
    39  	checkin(h addr) error
    40  	store(h addr, data io.Reader, size uint64) error
    41  }
    42  
    43  type fsTableCache struct {
    44  	dir   string
    45  	cache *sizecache.SizeCache
    46  	fd    *fdCache
    47  }
    48  
    49  func newFSTableCache(dir string, cacheSize uint64, maxOpenFds int) (*fsTableCache, error) {
    50  	ftc := &fsTableCache{dir: dir, fd: newFDCache(maxOpenFds)}
    51  	ftc.cache = sizecache.NewWithExpireCallback(cacheSize, func(elm interface{}) {
    52  		ftc.expire(elm.(addr))
    53  	})
    54  
    55  	err := ftc.init(maxOpenFds)
    56  
    57  	if err != nil {
    58  		return nil, err
    59  	}
    60  
    61  	return ftc, nil
    62  }
    63  
    64  func (ftc *fsTableCache) init(concurrency int) error {
    65  	type finfo struct {
    66  		path string
    67  		h    addr
    68  		size uint64
    69  	}
    70  	infos := make(chan finfo)
    71  	errc := make(chan error, 1)
    72  	go func() {
    73  		isTableFile := func(info os.FileInfo) bool {
    74  			return info.Mode().IsRegular() && ValidateAddr(info.Name())
    75  		}
    76  		isTempTableFile := func(info os.FileInfo) bool {
    77  			return info.Mode().IsRegular() && strings.HasPrefix(info.Name(), tempTablePrefix)
    78  		}
    79  		defer close(errc)
    80  		defer close(infos)
    81  		// No select needed for this send, since errc is buffered.
    82  		errc <- filepath.Walk(ftc.dir, func(path string, info os.FileInfo, err error) error {
    83  			if err != nil {
    84  				return err
    85  			}
    86  			if path == ftc.dir {
    87  				return nil
    88  			}
    89  			if isTempTableFile(info) {
    90  				// ignore failure to remove temp file
    91  				_ = os.Remove(path)
    92  				return nil
    93  			}
    94  			if !isTableFile(info) {
    95  				return errors.New(path + " is not a table file; cache dir must contain only table files")
    96  			}
    97  
    98  			ad, err := parseAddr(info.Name())
    99  
   100  			if err != nil {
   101  				return err
   102  			}
   103  
   104  			infos <- finfo{path, ad, uint64(info.Size())}
   105  			return nil
   106  		})
   107  	}()
   108  
   109  	ae := atomicerr.New()
   110  	wg := sync.WaitGroup{}
   111  	wg.Add(concurrency)
   112  	for i := 0; i < concurrency; i++ {
   113  		go func() {
   114  			defer wg.Done()
   115  			for info := range infos {
   116  				if ae.IsSet() {
   117  					break
   118  				}
   119  
   120  				ftc.cache.Add(info.h, info.size, true)
   121  				_, err := ftc.fd.RefFile(info.path)
   122  
   123  				if err != nil {
   124  					ae.SetIfError(err)
   125  					break
   126  				}
   127  
   128  				err = ftc.fd.UnrefFile(info.path)
   129  
   130  				if err != nil {
   131  					ae.SetIfError(err)
   132  					break
   133  				}
   134  			}
   135  		}()
   136  	}
   137  	wg.Wait()
   138  
   139  	err := <-errc
   140  
   141  	if err != nil {
   142  		return err
   143  	}
   144  
   145  	if err := ae.Get(); err != nil {
   146  		return err
   147  	}
   148  
   149  	return nil
   150  }
   151  
   152  func (ftc *fsTableCache) checkout(h addr) (io.ReaderAt, error) {
   153  	if _, ok := ftc.cache.Get(h); !ok {
   154  		return nil, nil
   155  	}
   156  
   157  	fd, err := ftc.fd.RefFile(filepath.Join(ftc.dir, h.String()))
   158  
   159  	if err != nil {
   160  		return nil, err
   161  	}
   162  
   163  	return fd, nil
   164  }
   165  
   166  func (ftc *fsTableCache) checkin(h addr) error {
   167  	return ftc.fd.UnrefFile(filepath.Join(ftc.dir, h.String()))
   168  }
   169  
   170  func (ftc *fsTableCache) store(h addr, data io.Reader, size uint64) error {
   171  	path := filepath.Join(ftc.dir, h.String())
   172  	tempName, err := func() (name string, ferr error) {
   173  		var temp *os.File
   174  		temp, ferr = tempfiles.MovableTempFileProvider.NewFile(ftc.dir, tempTablePrefix)
   175  
   176  		if ferr != nil {
   177  			return "", ferr
   178  		}
   179  
   180  		defer func() {
   181  			closeErr := temp.Close()
   182  
   183  			if ferr == nil {
   184  				ferr = closeErr
   185  			}
   186  		}()
   187  
   188  		_, ferr = io.Copy(temp, data)
   189  
   190  		if ferr != nil {
   191  			return "", ferr
   192  		}
   193  
   194  		return temp.Name(), nil
   195  	}()
   196  
   197  	if err != nil {
   198  		return err
   199  	}
   200  
   201  	err = ftc.fd.ShrinkCache()
   202  
   203  	if err != nil {
   204  		return err
   205  	}
   206  
   207  	err = os.Rename(tempName, path)
   208  
   209  	if err != nil {
   210  		return err
   211  	}
   212  
   213  	ftc.cache.Add(h, size, true)
   214  
   215  	// Prime the file in the fd cache ignore err
   216  	if _, err = ftc.fd.RefFile(path); err == nil {
   217  		err := ftc.fd.UnrefFile(path)
   218  
   219  		if err != nil {
   220  			return err
   221  		}
   222  	}
   223  
   224  	return nil
   225  }
   226  
   227  func (ftc *fsTableCache) expire(h addr) error {
   228  	return os.Remove(filepath.Join(ftc.dir, h.String()))
   229  }