github.com/oskarth/go-ethereum@v1.6.8-0.20191013093314-dac24a9d3494/swarm/storage/localstore.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 package storage 18 19 import ( 20 "context" 21 "path/filepath" 22 "sync" 23 24 "github.com/ethereum/go-ethereum/metrics" 25 "github.com/ethereum/go-ethereum/swarm/log" 26 "github.com/ethereum/go-ethereum/swarm/storage/mock" 27 ) 28 29 type LocalStoreParams struct { 30 *StoreParams 31 ChunkDbPath string 32 Validators []ChunkValidator `toml:"-"` 33 } 34 35 func NewDefaultLocalStoreParams() *LocalStoreParams { 36 return &LocalStoreParams{ 37 StoreParams: NewDefaultStoreParams(), 38 } 39 } 40 41 //this can only finally be set after all config options (file, cmd line, env vars) 42 //have been evaluated 43 func (p *LocalStoreParams) Init(path string) { 44 if p.ChunkDbPath == "" { 45 p.ChunkDbPath = filepath.Join(path, "chunks") 46 } 47 } 48 49 // LocalStore is a combination of inmemory db over a disk persisted db 50 // implements a Get/Put with fallback (caching) logic using any 2 ChunkStores 51 type LocalStore struct { 52 Validators []ChunkValidator 53 memStore *MemStore 54 DbStore *LDBStore 55 mu sync.Mutex 56 } 57 58 // This constructor uses MemStore and DbStore as components 59 func NewLocalStore(params *LocalStoreParams, mockStore *mock.NodeStore) (*LocalStore, error) { 60 ldbparams := NewLDBStoreParams(params.StoreParams, params.ChunkDbPath) 61 dbStore, err := NewMockDbStore(ldbparams, mockStore) 62 if err != nil { 63 return nil, err 64 } 65 return &LocalStore{ 66 memStore: NewMemStore(params.StoreParams, dbStore), 67 DbStore: dbStore, 68 Validators: params.Validators, 69 }, nil 70 } 71 72 func NewTestLocalStoreForAddr(params *LocalStoreParams) (*LocalStore, error) { 73 ldbparams := NewLDBStoreParams(params.StoreParams, params.ChunkDbPath) 74 dbStore, err := NewLDBStore(ldbparams) 75 if err != nil { 76 return nil, err 77 } 78 localStore := &LocalStore{ 79 memStore: NewMemStore(params.StoreParams, dbStore), 80 DbStore: dbStore, 81 Validators: params.Validators, 82 } 83 return localStore, nil 84 } 85 86 // isValid returns true if chunk passes any of the LocalStore Validators. 87 // isValid also returns true if LocalStore has no Validators. 88 func (ls *LocalStore) isValid(chunk Chunk) bool { 89 // by default chunks are valid. if we have 0 validators, then all chunks are valid. 90 valid := true 91 92 // ls.Validators contains a list of one validator per chunk type. 93 // if one validator succeeds, then the chunk is valid 94 for _, v := range ls.Validators { 95 if valid = v.Validate(chunk.Address(), chunk.Data()); valid { 96 break 97 } 98 } 99 return valid 100 } 101 102 // Put is responsible for doing validation and storage of the chunk 103 // by using configured ChunkValidators, MemStore and LDBStore. 104 // If the chunk is not valid, its GetErrored function will 105 // return ErrChunkInvalid. 106 // This method will check if the chunk is already in the MemStore 107 // and it will return it if it is. If there is an error from 108 // the MemStore.Get, it will be returned by calling GetErrored 109 // on the chunk. 110 // This method is responsible for closing Chunk.ReqC channel 111 // when the chunk is stored in memstore. 112 // After the LDBStore.Put, it is ensured that the MemStore 113 // contains the chunk with the same data, but nil ReqC channel. 114 func (ls *LocalStore) Put(ctx context.Context, chunk Chunk) error { 115 if !ls.isValid(chunk) { 116 return ErrChunkInvalid 117 } 118 119 log.Trace("localstore.put", "key", chunk.Address()) 120 ls.mu.Lock() 121 defer ls.mu.Unlock() 122 123 _, err := ls.memStore.Get(ctx, chunk.Address()) 124 if err == nil { 125 return nil 126 } 127 if err != nil && err != ErrChunkNotFound { 128 return err 129 } 130 ls.memStore.Put(ctx, chunk) 131 err = ls.DbStore.Put(ctx, chunk) 132 return err 133 } 134 135 // Get(chunk *Chunk) looks up a chunk in the local stores 136 // This method is blocking until the chunk is retrieved 137 // so additional timeout may be needed to wrap this call if 138 // ChunkStores are remote and can have long latency 139 func (ls *LocalStore) Get(ctx context.Context, addr Address) (chunk Chunk, err error) { 140 ls.mu.Lock() 141 defer ls.mu.Unlock() 142 143 return ls.get(ctx, addr) 144 } 145 146 func (ls *LocalStore) get(ctx context.Context, addr Address) (chunk Chunk, err error) { 147 chunk, err = ls.memStore.Get(ctx, addr) 148 149 if err != nil && err != ErrChunkNotFound { 150 metrics.GetOrRegisterCounter("localstore.get.error", nil).Inc(1) 151 return nil, err 152 } 153 154 if err == nil { 155 metrics.GetOrRegisterCounter("localstore.get.cachehit", nil).Inc(1) 156 return chunk, nil 157 } 158 159 metrics.GetOrRegisterCounter("localstore.get.cachemiss", nil).Inc(1) 160 chunk, err = ls.DbStore.Get(ctx, addr) 161 if err != nil { 162 metrics.GetOrRegisterCounter("localstore.get.error", nil).Inc(1) 163 return nil, err 164 } 165 166 ls.memStore.Put(ctx, chunk) 167 return chunk, nil 168 } 169 170 func (ls *LocalStore) FetchFunc(ctx context.Context, addr Address) func(context.Context) error { 171 ls.mu.Lock() 172 defer ls.mu.Unlock() 173 174 _, err := ls.get(ctx, addr) 175 if err == nil { 176 return nil 177 } 178 return func(context.Context) error { 179 return err 180 } 181 } 182 183 func (ls *LocalStore) BinIndex(po uint8) uint64 { 184 return ls.DbStore.BinIndex(po) 185 } 186 187 func (ls *LocalStore) Iterator(from uint64, to uint64, po uint8, f func(Address, uint64) bool) error { 188 return ls.DbStore.SyncIterator(from, to, po, f) 189 } 190 191 // Close the local store 192 func (ls *LocalStore) Close() { 193 ls.DbStore.Close() 194 } 195 196 // Migrate checks the datastore schema vs the runtime schema, and runs migrations if they don't match 197 func (ls *LocalStore) Migrate() error { 198 schema, err := ls.DbStore.GetSchema() 199 if err != nil { 200 log.Error(err.Error()) 201 return err 202 } 203 204 log.Debug("found schema", "schema", schema, "runtime-schema", CurrentDbSchema) 205 if schema != CurrentDbSchema { 206 // run migrations 207 208 if schema == "" { 209 log.Debug("running migrations for", "schema", schema, "runtime-schema", CurrentDbSchema) 210 211 // delete chunks that are not valid, i.e. chunks that do not pass any of the ls.Validators 212 ls.DbStore.Cleanup(func(c *chunk) bool { 213 return !ls.isValid(c) 214 }) 215 216 err := ls.DbStore.PutSchema(DbSchemaPurity) 217 if err != nil { 218 log.Error(err.Error()) 219 return err 220 } 221 } 222 } 223 224 return nil 225 }