github.com/sberex/go-sberex@v1.8.2-0.20181113200658-ed96ac38f7d7/swarm/storage/memstore.go (about) 1 // This file is part of the go-sberex library. The go-sberex library is 2 // free software: you can redistribute it and/or modify it under the terms 3 // of the GNU Lesser General Public License as published by the Free 4 // Software Foundation, either version 3 of the License, or (at your option) 5 // any later version. 6 // 7 // The go-sberex library is distributed in the hope that it will be useful, 8 // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser 10 // General Public License <http://www.gnu.org/licenses/> for more details. 11 12 // memory storage layer for the package blockhash 13 14 package storage 15 16 import ( 17 "fmt" 18 "sync" 19 20 "github.com/Sberex/go-sberex/log" 21 "github.com/Sberex/go-sberex/metrics" 22 ) 23 24 //metrics variables 25 var ( 26 memstorePutCounter = metrics.NewRegisteredCounter("storage.db.memstore.put.count", nil) 27 memstoreRemoveCounter = metrics.NewRegisteredCounter("storage.db.memstore.rm.count", nil) 28 ) 29 30 const ( 31 memTreeLW = 2 // log2(subtree count) of the subtrees 32 memTreeFLW = 14 // log2(subtree count) of the root layer 33 dbForceUpdateAccessCnt = 1000 34 defaultCacheCapacity = 5000 35 ) 36 37 type MemStore struct { 38 memtree *memTree 39 entryCnt, capacity uint // stored entries 40 accessCnt uint64 // access counter; oldest is thrown away when full 41 dbAccessCnt uint64 42 dbStore *DbStore 43 lock sync.Mutex 44 } 45 46 /* 47 a hash prefix subtree containing subtrees or one storage entry (but never both) 48 49 - access[0] stores the smallest (oldest) access count value in this subtree 50 - if it contains more subtrees and its subtree count is at least 4, access[1:2] 51 stores the smallest access count in the first and second halves of subtrees 52 (so that access[0] = min(access[1], access[2]) 53 - likewise, if subtree count is at least 8, 54 access[1] = min(access[3], access[4]) 55 access[2] = min(access[5], access[6]) 56 (access[] is a binary tree inside the multi-bit leveled hash tree) 57 */ 58 59 func NewMemStore(d *DbStore, capacity uint) (m *MemStore) { 60 m = &MemStore{} 61 m.memtree = newMemTree(memTreeFLW, nil, 0) 62 m.dbStore = d 63 m.setCapacity(capacity) 64 return 65 } 66 67 type memTree struct { 68 subtree []*memTree 69 parent *memTree 70 parentIdx uint 71 72 bits uint // log2(subtree count) 73 width uint // subtree count 74 75 entry *Chunk // if subtrees are present, entry should be nil 76 lastDBaccess uint64 77 access []uint64 78 } 79 80 func newMemTree(b uint, parent *memTree, pidx uint) (node *memTree) { 81 node = new(memTree) 82 node.bits = b 83 node.width = 1 << b 84 node.subtree = make([]*memTree, node.width) 85 node.access = make([]uint64, node.width-1) 86 node.parent = parent 87 node.parentIdx = pidx 88 if parent != nil { 89 parent.subtree[pidx] = node 90 } 91 92 return node 93 } 94 95 func (node *memTree) updateAccess(a uint64) { 96 aidx := uint(0) 97 var aa uint64 98 oa := node.access[0] 99 for node.access[aidx] == oa { 100 node.access[aidx] = a 101 if aidx > 0 { 102 aa = node.access[((aidx-1)^1)+1] 103 aidx = (aidx - 1) >> 1 104 } else { 105 pidx := node.parentIdx 106 node = node.parent 107 if node == nil { 108 return 109 } 110 nn := node.subtree[pidx^1] 111 if nn != nil { 112 aa = nn.access[0] 113 } else { 114 aa = 0 115 } 116 aidx = (node.width + pidx - 2) >> 1 117 } 118 119 if (aa != 0) && (aa < a) { 120 a = aa 121 } 122 } 123 } 124 125 func (s *MemStore) setCapacity(c uint) { 126 s.lock.Lock() 127 defer s.lock.Unlock() 128 129 for c < s.entryCnt { 130 s.removeOldest() 131 } 132 s.capacity = c 133 } 134 135 func (s *MemStore) Counter() uint { 136 return s.entryCnt 137 } 138 139 // entry (not its copy) is going to be in MemStore 140 func (s *MemStore) Put(entry *Chunk) { 141 if s.capacity == 0 { 142 return 143 } 144 145 s.lock.Lock() 146 defer s.lock.Unlock() 147 148 if s.entryCnt >= s.capacity { 149 s.removeOldest() 150 } 151 152 s.accessCnt++ 153 154 memstorePutCounter.Inc(1) 155 156 node := s.memtree 157 bitpos := uint(0) 158 for node.entry == nil { 159 l := entry.Key.bits(bitpos, node.bits) 160 st := node.subtree[l] 161 if st == nil { 162 st = newMemTree(memTreeLW, node, l) 163 bitpos += node.bits 164 node = st 165 break 166 } 167 bitpos += node.bits 168 node = st 169 } 170 171 if node.entry != nil { 172 173 if node.entry.Key.isEqual(entry.Key) { 174 node.updateAccess(s.accessCnt) 175 if entry.SData == nil { 176 entry.Size = node.entry.Size 177 entry.SData = node.entry.SData 178 } 179 if entry.Req == nil { 180 entry.Req = node.entry.Req 181 } 182 entry.C = node.entry.C 183 node.entry = entry 184 return 185 } 186 187 for node.entry != nil { 188 189 l := node.entry.Key.bits(bitpos, node.bits) 190 st := node.subtree[l] 191 if st == nil { 192 st = newMemTree(memTreeLW, node, l) 193 } 194 st.entry = node.entry 195 node.entry = nil 196 st.updateAccess(node.access[0]) 197 198 l = entry.Key.bits(bitpos, node.bits) 199 st = node.subtree[l] 200 if st == nil { 201 st = newMemTree(memTreeLW, node, l) 202 } 203 bitpos += node.bits 204 node = st 205 206 } 207 } 208 209 node.entry = entry 210 node.lastDBaccess = s.dbAccessCnt 211 node.updateAccess(s.accessCnt) 212 s.entryCnt++ 213 } 214 215 func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) { 216 s.lock.Lock() 217 defer s.lock.Unlock() 218 219 node := s.memtree 220 bitpos := uint(0) 221 for node.entry == nil { 222 l := hash.bits(bitpos, node.bits) 223 st := node.subtree[l] 224 if st == nil { 225 return nil, notFound 226 } 227 bitpos += node.bits 228 node = st 229 } 230 231 if node.entry.Key.isEqual(hash) { 232 s.accessCnt++ 233 node.updateAccess(s.accessCnt) 234 chunk = node.entry 235 if s.dbAccessCnt-node.lastDBaccess > dbForceUpdateAccessCnt { 236 s.dbAccessCnt++ 237 node.lastDBaccess = s.dbAccessCnt 238 if s.dbStore != nil { 239 s.dbStore.updateAccessCnt(hash) 240 } 241 } 242 } else { 243 err = notFound 244 } 245 246 return 247 } 248 249 func (s *MemStore) removeOldest() { 250 node := s.memtree 251 252 for node.entry == nil { 253 254 aidx := uint(0) 255 av := node.access[aidx] 256 257 for aidx < node.width/2-1 { 258 if av == node.access[aidx*2+1] { 259 node.access[aidx] = node.access[aidx*2+2] 260 aidx = aidx*2 + 1 261 } else if av == node.access[aidx*2+2] { 262 node.access[aidx] = node.access[aidx*2+1] 263 aidx = aidx*2 + 2 264 } else { 265 panic(nil) 266 } 267 } 268 pidx := aidx*2 + 2 - node.width 269 if (node.subtree[pidx] != nil) && (av == node.subtree[pidx].access[0]) { 270 if node.subtree[pidx+1] != nil { 271 node.access[aidx] = node.subtree[pidx+1].access[0] 272 } else { 273 node.access[aidx] = 0 274 } 275 } else if (node.subtree[pidx+1] != nil) && (av == node.subtree[pidx+1].access[0]) { 276 if node.subtree[pidx] != nil { 277 node.access[aidx] = node.subtree[pidx].access[0] 278 } else { 279 node.access[aidx] = 0 280 } 281 pidx++ 282 } else { 283 panic(nil) 284 } 285 286 //fmt.Println(pidx) 287 node = node.subtree[pidx] 288 289 } 290 291 if node.entry.dbStored != nil { 292 log.Trace(fmt.Sprintf("Memstore Clean: Waiting for chunk %v to be saved", node.entry.Key.Log())) 293 <-node.entry.dbStored 294 log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v saved to DBStore. Ready to clear from mem.", node.entry.Key.Log())) 295 } else { 296 log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v already in DB. Ready to delete.", node.entry.Key.Log())) 297 } 298 299 if node.entry.SData != nil { 300 memstoreRemoveCounter.Inc(1) 301 node.entry = nil 302 s.entryCnt-- 303 } 304 305 node.access[0] = 0 306 307 //--- 308 309 aidx := uint(0) 310 for { 311 aa := node.access[aidx] 312 if aidx > 0 { 313 aidx = (aidx - 1) >> 1 314 } else { 315 pidx := node.parentIdx 316 node = node.parent 317 if node == nil { 318 return 319 } 320 aidx = (node.width + pidx - 2) >> 1 321 } 322 if (aa != 0) && ((aa < node.access[aidx]) || (node.access[aidx] == 0)) { 323 node.access[aidx] = aa 324 } 325 } 326 } 327 328 // Close memstore 329 func (s *MemStore) Close() {}