github.com/xfond/eth-implementation@v1.8.9-0.20180514135602-f6bc65fc6811/swarm/storage/memstore.go (about) 1 // Copyright 2016 The go-ethereum Authors 2 // This file is part of the go-ethereum library. 3 // 4 // The go-ethereum library is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Lesser General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // The go-ethereum library is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Lesser General Public License for more details. 13 // 14 // You should have received a copy of the GNU Lesser General Public License 15 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. 16 17 // memory storage layer for the package blockhash 18 19 package storage 20 21 import ( 22 "fmt" 23 "sync" 24 25 "github.com/ethereum/go-ethereum/log" 26 "github.com/ethereum/go-ethereum/metrics" 27 ) 28 29 //metrics variables 30 var ( 31 memstorePutCounter = metrics.NewRegisteredCounter("storage.db.memstore.put.count", nil) 32 memstoreRemoveCounter = metrics.NewRegisteredCounter("storage.db.memstore.rm.count", nil) 33 ) 34 35 const ( 36 memTreeLW = 2 // log2(subtree count) of the subtrees 37 memTreeFLW = 14 // log2(subtree count) of the root layer 38 dbForceUpdateAccessCnt = 1000 39 defaultCacheCapacity = 5000 40 ) 41 42 type MemStore struct { 43 memtree *memTree 44 entryCnt, capacity uint // stored entries 45 accessCnt uint64 // access counter; oldest is thrown away when full 46 dbAccessCnt uint64 47 dbStore *DbStore 48 lock sync.Mutex 49 } 50 51 /* 52 a hash prefix subtree containing subtrees or one storage entry (but never both) 53 54 - access[0] stores the smallest (oldest) access count value in this subtree 55 - if it contains more subtrees and its subtree count is at least 4, access[1:2] 56 stores the smallest access count in the first and second halves of subtrees 57 (so that access[0] = min(access[1], access[2]) 58 - likewise, if subtree count is at least 8, 59 access[1] = min(access[3], access[4]) 60 access[2] = min(access[5], access[6]) 61 (access[] is a binary tree inside the multi-bit leveled hash tree) 62 */ 63 64 func NewMemStore(d *DbStore, capacity uint) (m *MemStore) { 65 m = &MemStore{} 66 m.memtree = newMemTree(memTreeFLW, nil, 0) 67 m.dbStore = d 68 m.setCapacity(capacity) 69 return 70 } 71 72 type memTree struct { 73 subtree []*memTree 74 parent *memTree 75 parentIdx uint 76 77 bits uint // log2(subtree count) 78 width uint // subtree count 79 80 entry *Chunk // if subtrees are present, entry should be nil 81 lastDBaccess uint64 82 access []uint64 83 } 84 85 func newMemTree(b uint, parent *memTree, pidx uint) (node *memTree) { 86 node = new(memTree) 87 node.bits = b 88 node.width = 1 << b 89 node.subtree = make([]*memTree, node.width) 90 node.access = make([]uint64, node.width-1) 91 node.parent = parent 92 node.parentIdx = pidx 93 if parent != nil { 94 parent.subtree[pidx] = node 95 } 96 97 return node 98 } 99 100 func (node *memTree) updateAccess(a uint64) { 101 aidx := uint(0) 102 var aa uint64 103 oa := node.access[0] 104 for node.access[aidx] == oa { 105 node.access[aidx] = a 106 if aidx > 0 { 107 aa = node.access[((aidx-1)^1)+1] 108 aidx = (aidx - 1) >> 1 109 } else { 110 pidx := node.parentIdx 111 node = node.parent 112 if node == nil { 113 return 114 } 115 nn := node.subtree[pidx^1] 116 if nn != nil { 117 aa = nn.access[0] 118 } else { 119 aa = 0 120 } 121 aidx = (node.width + pidx - 2) >> 1 122 } 123 124 if (aa != 0) && (aa < a) { 125 a = aa 126 } 127 } 128 } 129 130 func (s *MemStore) setCapacity(c uint) { 131 s.lock.Lock() 132 defer s.lock.Unlock() 133 134 for c < s.entryCnt { 135 s.removeOldest() 136 } 137 s.capacity = c 138 } 139 140 func (s *MemStore) Counter() uint { 141 return s.entryCnt 142 } 143 144 // entry (not its copy) is going to be in MemStore 145 func (s *MemStore) Put(entry *Chunk) { 146 if s.capacity == 0 { 147 return 148 } 149 150 s.lock.Lock() 151 defer s.lock.Unlock() 152 153 if s.entryCnt >= s.capacity { 154 s.removeOldest() 155 } 156 157 s.accessCnt++ 158 159 memstorePutCounter.Inc(1) 160 161 node := s.memtree 162 bitpos := uint(0) 163 for node.entry == nil { 164 l := entry.Key.bits(bitpos, node.bits) 165 st := node.subtree[l] 166 if st == nil { 167 st = newMemTree(memTreeLW, node, l) 168 bitpos += node.bits 169 node = st 170 break 171 } 172 bitpos += node.bits 173 node = st 174 } 175 176 if node.entry != nil { 177 178 if node.entry.Key.isEqual(entry.Key) { 179 node.updateAccess(s.accessCnt) 180 if entry.SData == nil { 181 entry.Size = node.entry.Size 182 entry.SData = node.entry.SData 183 } 184 if entry.Req == nil { 185 entry.Req = node.entry.Req 186 } 187 entry.C = node.entry.C 188 node.entry = entry 189 return 190 } 191 192 for node.entry != nil { 193 194 l := node.entry.Key.bits(bitpos, node.bits) 195 st := node.subtree[l] 196 if st == nil { 197 st = newMemTree(memTreeLW, node, l) 198 } 199 st.entry = node.entry 200 node.entry = nil 201 st.updateAccess(node.access[0]) 202 203 l = entry.Key.bits(bitpos, node.bits) 204 st = node.subtree[l] 205 if st == nil { 206 st = newMemTree(memTreeLW, node, l) 207 } 208 bitpos += node.bits 209 node = st 210 211 } 212 } 213 214 node.entry = entry 215 node.lastDBaccess = s.dbAccessCnt 216 node.updateAccess(s.accessCnt) 217 s.entryCnt++ 218 } 219 220 func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) { 221 s.lock.Lock() 222 defer s.lock.Unlock() 223 224 node := s.memtree 225 bitpos := uint(0) 226 for node.entry == nil { 227 l := hash.bits(bitpos, node.bits) 228 st := node.subtree[l] 229 if st == nil { 230 return nil, notFound 231 } 232 bitpos += node.bits 233 node = st 234 } 235 236 if node.entry.Key.isEqual(hash) { 237 s.accessCnt++ 238 node.updateAccess(s.accessCnt) 239 chunk = node.entry 240 if s.dbAccessCnt-node.lastDBaccess > dbForceUpdateAccessCnt { 241 s.dbAccessCnt++ 242 node.lastDBaccess = s.dbAccessCnt 243 if s.dbStore != nil { 244 s.dbStore.updateAccessCnt(hash) 245 } 246 } 247 } else { 248 err = notFound 249 } 250 251 return 252 } 253 254 func (s *MemStore) removeOldest() { 255 node := s.memtree 256 257 for node.entry == nil { 258 259 aidx := uint(0) 260 av := node.access[aidx] 261 262 for aidx < node.width/2-1 { 263 if av == node.access[aidx*2+1] { 264 node.access[aidx] = node.access[aidx*2+2] 265 aidx = aidx*2 + 1 266 } else if av == node.access[aidx*2+2] { 267 node.access[aidx] = node.access[aidx*2+1] 268 aidx = aidx*2 + 2 269 } else { 270 panic(nil) 271 } 272 } 273 pidx := aidx*2 + 2 - node.width 274 if (node.subtree[pidx] != nil) && (av == node.subtree[pidx].access[0]) { 275 if node.subtree[pidx+1] != nil { 276 node.access[aidx] = node.subtree[pidx+1].access[0] 277 } else { 278 node.access[aidx] = 0 279 } 280 } else if (node.subtree[pidx+1] != nil) && (av == node.subtree[pidx+1].access[0]) { 281 if node.subtree[pidx] != nil { 282 node.access[aidx] = node.subtree[pidx].access[0] 283 } else { 284 node.access[aidx] = 0 285 } 286 pidx++ 287 } else { 288 panic(nil) 289 } 290 291 //fmt.Println(pidx) 292 node = node.subtree[pidx] 293 294 } 295 296 if node.entry.dbStored != nil { 297 log.Trace(fmt.Sprintf("Memstore Clean: Waiting for chunk %v to be saved", node.entry.Key.Log())) 298 <-node.entry.dbStored 299 log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v saved to DBStore. Ready to clear from mem.", node.entry.Key.Log())) 300 } else { 301 log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v already in DB. Ready to delete.", node.entry.Key.Log())) 302 } 303 304 if node.entry.SData != nil { 305 memstoreRemoveCounter.Inc(1) 306 node.entry = nil 307 s.entryCnt-- 308 } 309 310 node.access[0] = 0 311 312 //--- 313 314 aidx := uint(0) 315 for { 316 aa := node.access[aidx] 317 if aidx > 0 { 318 aidx = (aidx - 1) >> 1 319 } else { 320 pidx := node.parentIdx 321 node = node.parent 322 if node == nil { 323 return 324 } 325 aidx = (node.width + pidx - 2) >> 1 326 } 327 if (aa != 0) && ((aa < node.access[aidx]) || (node.access[aidx] == 0)) { 328 node.access[aidx] = aa 329 } 330 } 331 } 332 333 // Close memstore 334 func (s *MemStore) Close() {}