github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/store/nbs/fs_table_cache.go (about) 1 // Copyright 2019 Dolthub, Inc. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 // 15 // This file incorporates work covered by the following copyright and 16 // permission notice: 17 // 18 // Copyright 2017 Attic Labs, Inc. All rights reserved. 19 // Licensed under the Apache License, version 2.0: 20 // http://www.apache.org/licenses/LICENSE-2.0 21 22 package nbs 23 24 import ( 25 "errors" 26 "io" 27 "os" 28 "path/filepath" 29 "strings" 30 "sync" 31 32 "github.com/dolthub/dolt/go/store/atomicerr" 33 "github.com/dolthub/dolt/go/store/util/sizecache" 34 "github.com/dolthub/dolt/go/store/util/tempfiles" 35 ) 36 37 type tableCache interface { 38 checkout(h addr) (io.ReaderAt, error) 39 checkin(h addr) error 40 store(h addr, data io.Reader, size uint64) error 41 } 42 43 type fsTableCache struct { 44 dir string 45 cache *sizecache.SizeCache 46 fd *fdCache 47 } 48 49 func newFSTableCache(dir string, cacheSize uint64, maxOpenFds int) (*fsTableCache, error) { 50 ftc := &fsTableCache{dir: dir, fd: newFDCache(maxOpenFds)} 51 ftc.cache = sizecache.NewWithExpireCallback(cacheSize, func(elm interface{}) { 52 ftc.expire(elm.(addr)) 53 }) 54 55 err := ftc.init(maxOpenFds) 56 57 if err != nil { 58 return nil, err 59 } 60 61 return ftc, nil 62 } 63 64 func (ftc *fsTableCache) init(concurrency int) error { 65 type finfo struct { 66 path string 67 h addr 68 size uint64 69 } 70 infos := make(chan finfo) 71 errc := make(chan error, 1) 72 go func() { 73 isTableFile := func(info os.FileInfo) bool { 74 return info.Mode().IsRegular() && ValidateAddr(info.Name()) 75 } 76 isTempTableFile := func(info os.FileInfo) bool { 77 return info.Mode().IsRegular() && strings.HasPrefix(info.Name(), tempTablePrefix) 78 } 79 defer close(errc) 80 defer close(infos) 81 // No select needed for this send, since errc is buffered. 82 errc <- filepath.Walk(ftc.dir, func(path string, info os.FileInfo, err error) error { 83 if err != nil { 84 return err 85 } 86 if path == ftc.dir { 87 return nil 88 } 89 if isTempTableFile(info) { 90 // ignore failure to remove temp file 91 _ = os.Remove(path) 92 return nil 93 } 94 if !isTableFile(info) { 95 return errors.New(path + " is not a table file; cache dir must contain only table files") 96 } 97 98 ad, err := parseAddr(info.Name()) 99 100 if err != nil { 101 return err 102 } 103 104 infos <- finfo{path, ad, uint64(info.Size())} 105 return nil 106 }) 107 }() 108 109 ae := atomicerr.New() 110 wg := sync.WaitGroup{} 111 wg.Add(concurrency) 112 for i := 0; i < concurrency; i++ { 113 go func() { 114 defer wg.Done() 115 for info := range infos { 116 if ae.IsSet() { 117 break 118 } 119 120 ftc.cache.Add(info.h, info.size, true) 121 _, err := ftc.fd.RefFile(info.path) 122 123 if err != nil { 124 ae.SetIfError(err) 125 break 126 } 127 128 err = ftc.fd.UnrefFile(info.path) 129 130 if err != nil { 131 ae.SetIfError(err) 132 break 133 } 134 } 135 }() 136 } 137 wg.Wait() 138 139 err := <-errc 140 141 if err != nil { 142 return err 143 } 144 145 if err := ae.Get(); err != nil { 146 return err 147 } 148 149 return nil 150 } 151 152 func (ftc *fsTableCache) checkout(h addr) (io.ReaderAt, error) { 153 if _, ok := ftc.cache.Get(h); !ok { 154 return nil, nil 155 } 156 157 fd, err := ftc.fd.RefFile(filepath.Join(ftc.dir, h.String())) 158 159 if err != nil { 160 return nil, err 161 } 162 163 return fd, nil 164 } 165 166 func (ftc *fsTableCache) checkin(h addr) error { 167 return ftc.fd.UnrefFile(filepath.Join(ftc.dir, h.String())) 168 } 169 170 func (ftc *fsTableCache) store(h addr, data io.Reader, size uint64) error { 171 path := filepath.Join(ftc.dir, h.String()) 172 tempName, err := func() (name string, ferr error) { 173 var temp *os.File 174 temp, ferr = tempfiles.MovableTempFileProvider.NewFile(ftc.dir, tempTablePrefix) 175 176 if ferr != nil { 177 return "", ferr 178 } 179 180 defer func() { 181 closeErr := temp.Close() 182 183 if ferr == nil { 184 ferr = closeErr 185 } 186 }() 187 188 _, ferr = io.Copy(temp, data) 189 190 if ferr != nil { 191 return "", ferr 192 } 193 194 return temp.Name(), nil 195 }() 196 197 if err != nil { 198 return err 199 } 200 201 err = ftc.fd.ShrinkCache() 202 203 if err != nil { 204 return err 205 } 206 207 err = os.Rename(tempName, path) 208 209 if err != nil { 210 return err 211 } 212 213 ftc.cache.Add(h, size, true) 214 215 // Prime the file in the fd cache ignore err 216 if _, err = ftc.fd.RefFile(path); err == nil { 217 err := ftc.fd.UnrefFile(path) 218 219 if err != nil { 220 return err 221 } 222 } 223 224 return nil 225 } 226 227 func (ftc *fsTableCache) expire(h addr) error { 228 return os.Remove(filepath.Join(ftc.dir, h.String())) 229 }