github.com/sberex/go-sberex@v1.8.2-0.20181113200658-ed96ac38f7d7/swarm/storage/dbstore.go (about) 1 // This file is part of the go-sberex library. The go-sberex library is 2 // free software: you can redistribute it and/or modify it under the terms 3 // of the GNU Lesser General Public License as published by the Free 4 // Software Foundation, either version 3 of the License, or (at your option) 5 // any later version. 6 // 7 // The go-sberex library is distributed in the hope that it will be useful, 8 // but WITHOUT ANY WARRANTY; without even the implied warranty of 9 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser 10 // General Public License <http://www.gnu.org/licenses/> for more details. 11 12 // disk storage layer for the package bzz 13 // DbStore implements the ChunkStore interface and is used by the DPA as 14 // persistent storage of chunks 15 // it implements purging based on access count allowing for external control of 16 // max capacity 17 18 package storage 19 20 import ( 21 "archive/tar" 22 "bytes" 23 "encoding/binary" 24 "encoding/hex" 25 "fmt" 26 "io" 27 "io/ioutil" 28 "sync" 29 30 "github.com/Sberex/go-sberex/log" 31 "github.com/Sberex/go-sberex/metrics" 32 "github.com/Sberex/go-sberex/rlp" 33 "github.com/syndtr/goleveldb/leveldb" 34 "github.com/syndtr/goleveldb/leveldb/iterator" 35 ) 36 37 //metrics variables 38 var ( 39 gcCounter = metrics.NewRegisteredCounter("storage.db.dbstore.gc.count", nil) 40 dbStoreDeleteCounter = metrics.NewRegisteredCounter("storage.db.dbstore.rm.count", nil) 41 ) 42 43 const ( 44 defaultDbCapacity = 5000000 45 defaultRadius = 0 // not yet used 46 47 gcArraySize = 10000 48 gcArrayFreeRatio = 0.1 49 50 // key prefixes for leveldb storage 51 kpIndex = 0 52 kpData = 1 53 ) 54 55 var ( 56 keyAccessCnt = []byte{2} 57 keyEntryCnt = []byte{3} 58 keyDataIdx = []byte{4} 59 keyGCPos = []byte{5} 60 ) 61 62 type gcItem struct { 63 idx uint64 64 value uint64 65 idxKey []byte 66 } 67 68 type DbStore struct { 69 db *LDBDatabase 70 71 // this should be stored in db, accessed transactionally 72 entryCnt, accessCnt, dataIdx, capacity uint64 73 74 gcPos, gcStartPos []byte 75 gcArray []*gcItem 76 77 hashfunc SwarmHasher 78 79 lock sync.Mutex 80 } 81 82 func NewDbStore(path string, hash SwarmHasher, capacity uint64, radius int) (s *DbStore, err error) { 83 s = new(DbStore) 84 85 s.hashfunc = hash 86 87 s.db, err = NewLDBDatabase(path) 88 if err != nil { 89 return 90 } 91 92 s.setCapacity(capacity) 93 94 s.gcStartPos = make([]byte, 1) 95 s.gcStartPos[0] = kpIndex 96 s.gcArray = make([]*gcItem, gcArraySize) 97 98 data, _ := s.db.Get(keyEntryCnt) 99 s.entryCnt = BytesToU64(data) 100 data, _ = s.db.Get(keyAccessCnt) 101 s.accessCnt = BytesToU64(data) 102 data, _ = s.db.Get(keyDataIdx) 103 s.dataIdx = BytesToU64(data) 104 s.gcPos, _ = s.db.Get(keyGCPos) 105 if s.gcPos == nil { 106 s.gcPos = s.gcStartPos 107 } 108 return 109 } 110 111 type dpaDBIndex struct { 112 Idx uint64 113 Access uint64 114 } 115 116 func BytesToU64(data []byte) uint64 { 117 if len(data) < 8 { 118 return 0 119 } 120 return binary.LittleEndian.Uint64(data) 121 } 122 123 func U64ToBytes(val uint64) []byte { 124 data := make([]byte, 8) 125 binary.LittleEndian.PutUint64(data, val) 126 return data 127 } 128 129 func getIndexGCValue(index *dpaDBIndex) uint64 { 130 return index.Access 131 } 132 133 func (s *DbStore) updateIndexAccess(index *dpaDBIndex) { 134 index.Access = s.accessCnt 135 } 136 137 func getIndexKey(hash Key) []byte { 138 HashSize := len(hash) 139 key := make([]byte, HashSize+1) 140 key[0] = 0 141 copy(key[1:], hash[:]) 142 return key 143 } 144 145 func getDataKey(idx uint64) []byte { 146 key := make([]byte, 9) 147 key[0] = 1 148 binary.BigEndian.PutUint64(key[1:9], idx) 149 150 return key 151 } 152 153 func encodeIndex(index *dpaDBIndex) []byte { 154 data, _ := rlp.EncodeToBytes(index) 155 return data 156 } 157 158 func encodeData(chunk *Chunk) []byte { 159 return chunk.SData 160 } 161 162 func decodeIndex(data []byte, index *dpaDBIndex) { 163 dec := rlp.NewStream(bytes.NewReader(data), 0) 164 dec.Decode(index) 165 } 166 167 func decodeData(data []byte, chunk *Chunk) { 168 chunk.SData = data 169 chunk.Size = int64(binary.LittleEndian.Uint64(data[0:8])) 170 } 171 172 func gcListPartition(list []*gcItem, left int, right int, pivotIndex int) int { 173 pivotValue := list[pivotIndex].value 174 dd := list[pivotIndex] 175 list[pivotIndex] = list[right] 176 list[right] = dd 177 storeIndex := left 178 for i := left; i < right; i++ { 179 if list[i].value < pivotValue { 180 dd = list[storeIndex] 181 list[storeIndex] = list[i] 182 list[i] = dd 183 storeIndex++ 184 } 185 } 186 dd = list[storeIndex] 187 list[storeIndex] = list[right] 188 list[right] = dd 189 return storeIndex 190 } 191 192 func gcListSelect(list []*gcItem, left int, right int, n int) int { 193 if left == right { 194 return left 195 } 196 pivotIndex := (left + right) / 2 197 pivotIndex = gcListPartition(list, left, right, pivotIndex) 198 if n == pivotIndex { 199 return n 200 } else { 201 if n < pivotIndex { 202 return gcListSelect(list, left, pivotIndex-1, n) 203 } else { 204 return gcListSelect(list, pivotIndex+1, right, n) 205 } 206 } 207 } 208 209 func (s *DbStore) collectGarbage(ratio float32) { 210 it := s.db.NewIterator() 211 it.Seek(s.gcPos) 212 if it.Valid() { 213 s.gcPos = it.Key() 214 } else { 215 s.gcPos = nil 216 } 217 gcnt := 0 218 219 for (gcnt < gcArraySize) && (uint64(gcnt) < s.entryCnt) { 220 221 if (s.gcPos == nil) || (s.gcPos[0] != kpIndex) { 222 it.Seek(s.gcStartPos) 223 if it.Valid() { 224 s.gcPos = it.Key() 225 } else { 226 s.gcPos = nil 227 } 228 } 229 230 if (s.gcPos == nil) || (s.gcPos[0] != kpIndex) { 231 break 232 } 233 234 gci := new(gcItem) 235 gci.idxKey = s.gcPos 236 var index dpaDBIndex 237 decodeIndex(it.Value(), &index) 238 gci.idx = index.Idx 239 // the smaller, the more likely to be gc'd 240 gci.value = getIndexGCValue(&index) 241 s.gcArray[gcnt] = gci 242 gcnt++ 243 it.Next() 244 if it.Valid() { 245 s.gcPos = it.Key() 246 } else { 247 s.gcPos = nil 248 } 249 } 250 it.Release() 251 252 cutidx := gcListSelect(s.gcArray, 0, gcnt-1, int(float32(gcnt)*ratio)) 253 cutval := s.gcArray[cutidx].value 254 255 // fmt.Print(gcnt, " ", s.entryCnt, " ") 256 257 // actual gc 258 for i := 0; i < gcnt; i++ { 259 if s.gcArray[i].value <= cutval { 260 gcCounter.Inc(1) 261 s.delete(s.gcArray[i].idx, s.gcArray[i].idxKey) 262 } 263 } 264 265 // fmt.Println(s.entryCnt) 266 267 s.db.Put(keyGCPos, s.gcPos) 268 } 269 270 // Export writes all chunks from the store to a tar archive, returning the 271 // number of chunks written. 272 func (s *DbStore) Export(out io.Writer) (int64, error) { 273 tw := tar.NewWriter(out) 274 defer tw.Close() 275 276 it := s.db.NewIterator() 277 defer it.Release() 278 var count int64 279 for ok := it.Seek([]byte{kpIndex}); ok; ok = it.Next() { 280 key := it.Key() 281 if (key == nil) || (key[0] != kpIndex) { 282 break 283 } 284 285 var index dpaDBIndex 286 decodeIndex(it.Value(), &index) 287 288 data, err := s.db.Get(getDataKey(index.Idx)) 289 if err != nil { 290 log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key[:], err)) 291 continue 292 } 293 294 hdr := &tar.Header{ 295 Name: hex.EncodeToString(key[1:]), 296 Mode: 0644, 297 Size: int64(len(data)), 298 } 299 if err := tw.WriteHeader(hdr); err != nil { 300 return count, err 301 } 302 if _, err := tw.Write(data); err != nil { 303 return count, err 304 } 305 count++ 306 } 307 308 return count, nil 309 } 310 311 // Import reads chunks into the store from a tar archive, returning the number 312 // of chunks read. 313 func (s *DbStore) Import(in io.Reader) (int64, error) { 314 tr := tar.NewReader(in) 315 316 var count int64 317 for { 318 hdr, err := tr.Next() 319 if err == io.EOF { 320 break 321 } else if err != nil { 322 return count, err 323 } 324 325 if len(hdr.Name) != 64 { 326 log.Warn("ignoring non-chunk file", "name", hdr.Name) 327 continue 328 } 329 330 key, err := hex.DecodeString(hdr.Name) 331 if err != nil { 332 log.Warn("ignoring invalid chunk file", "name", hdr.Name, "err", err) 333 continue 334 } 335 336 data, err := ioutil.ReadAll(tr) 337 if err != nil { 338 return count, err 339 } 340 341 s.Put(&Chunk{Key: key, SData: data}) 342 count++ 343 } 344 345 return count, nil 346 } 347 348 func (s *DbStore) Cleanup() { 349 //Iterates over the database and checks that there are no faulty chunks 350 it := s.db.NewIterator() 351 startPosition := []byte{kpIndex} 352 it.Seek(startPosition) 353 var key []byte 354 var errorsFound, total int 355 for it.Valid() { 356 key = it.Key() 357 if (key == nil) || (key[0] != kpIndex) { 358 break 359 } 360 total++ 361 var index dpaDBIndex 362 decodeIndex(it.Value(), &index) 363 364 data, err := s.db.Get(getDataKey(index.Idx)) 365 if err != nil { 366 log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key[:], err)) 367 s.delete(index.Idx, getIndexKey(key[1:])) 368 errorsFound++ 369 } else { 370 hasher := s.hashfunc() 371 hasher.Write(data) 372 hash := hasher.Sum(nil) 373 if !bytes.Equal(hash, key[1:]) { 374 log.Warn(fmt.Sprintf("Found invalid chunk. Hash mismatch. hash=%x, key=%x", hash, key[:])) 375 s.delete(index.Idx, getIndexKey(key[1:])) 376 errorsFound++ 377 } 378 } 379 it.Next() 380 } 381 it.Release() 382 log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total)) 383 } 384 385 func (s *DbStore) delete(idx uint64, idxKey []byte) { 386 batch := new(leveldb.Batch) 387 batch.Delete(idxKey) 388 batch.Delete(getDataKey(idx)) 389 dbStoreDeleteCounter.Inc(1) 390 s.entryCnt-- 391 batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) 392 s.db.Write(batch) 393 } 394 395 func (s *DbStore) Counter() uint64 { 396 s.lock.Lock() 397 defer s.lock.Unlock() 398 return s.dataIdx 399 } 400 401 func (s *DbStore) Put(chunk *Chunk) { 402 s.lock.Lock() 403 defer s.lock.Unlock() 404 405 ikey := getIndexKey(chunk.Key) 406 var index dpaDBIndex 407 408 if s.tryAccessIdx(ikey, &index) { 409 if chunk.dbStored != nil { 410 close(chunk.dbStored) 411 } 412 log.Trace(fmt.Sprintf("Storing to DB: chunk already exists, only update access")) 413 return // already exists, only update access 414 } 415 416 data := encodeData(chunk) 417 //data := ethutil.Encode([]interface{}{entry}) 418 419 if s.entryCnt >= s.capacity { 420 s.collectGarbage(gcArrayFreeRatio) 421 } 422 423 batch := new(leveldb.Batch) 424 425 batch.Put(getDataKey(s.dataIdx), data) 426 427 index.Idx = s.dataIdx 428 s.updateIndexAccess(&index) 429 430 idata := encodeIndex(&index) 431 batch.Put(ikey, idata) 432 433 batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) 434 s.entryCnt++ 435 batch.Put(keyDataIdx, U64ToBytes(s.dataIdx)) 436 s.dataIdx++ 437 batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) 438 s.accessCnt++ 439 440 s.db.Write(batch) 441 if chunk.dbStored != nil { 442 close(chunk.dbStored) 443 } 444 log.Trace(fmt.Sprintf("DbStore.Put: %v. db storage counter: %v ", chunk.Key.Log(), s.dataIdx)) 445 } 446 447 // try to find index; if found, update access cnt and return true 448 func (s *DbStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool { 449 idata, err := s.db.Get(ikey) 450 if err != nil { 451 return false 452 } 453 decodeIndex(idata, index) 454 455 batch := new(leveldb.Batch) 456 457 batch.Put(keyAccessCnt, U64ToBytes(s.accessCnt)) 458 s.accessCnt++ 459 s.updateIndexAccess(index) 460 idata = encodeIndex(index) 461 batch.Put(ikey, idata) 462 463 s.db.Write(batch) 464 465 return true 466 } 467 468 func (s *DbStore) Get(key Key) (chunk *Chunk, err error) { 469 s.lock.Lock() 470 defer s.lock.Unlock() 471 472 var index dpaDBIndex 473 474 if s.tryAccessIdx(getIndexKey(key), &index) { 475 var data []byte 476 data, err = s.db.Get(getDataKey(index.Idx)) 477 if err != nil { 478 log.Trace(fmt.Sprintf("DBStore: Chunk %v found but could not be accessed: %v", key.Log(), err)) 479 s.delete(index.Idx, getIndexKey(key)) 480 return 481 } 482 483 hasher := s.hashfunc() 484 hasher.Write(data) 485 hash := hasher.Sum(nil) 486 if !bytes.Equal(hash, key) { 487 s.delete(index.Idx, getIndexKey(key)) 488 log.Warn("Invalid Chunk in Database. Please repair with command: 'swarm cleandb'") 489 } 490 491 chunk = &Chunk{ 492 Key: key, 493 } 494 decodeData(data, chunk) 495 } else { 496 err = notFound 497 } 498 499 return 500 501 } 502 503 func (s *DbStore) updateAccessCnt(key Key) { 504 505 s.lock.Lock() 506 defer s.lock.Unlock() 507 508 var index dpaDBIndex 509 s.tryAccessIdx(getIndexKey(key), &index) // result_chn == nil, only update access cnt 510 511 } 512 513 func (s *DbStore) setCapacity(c uint64) { 514 515 s.lock.Lock() 516 defer s.lock.Unlock() 517 518 s.capacity = c 519 520 if s.entryCnt > c { 521 ratio := float32(1.01) - float32(c)/float32(s.entryCnt) 522 if ratio < gcArrayFreeRatio { 523 ratio = gcArrayFreeRatio 524 } 525 if ratio > 1 { 526 ratio = 1 527 } 528 for s.entryCnt > c { 529 s.collectGarbage(ratio) 530 } 531 } 532 } 533 534 func (s *DbStore) Close() { 535 s.db.Close() 536 } 537 538 // describes a section of the DbStore representing the unsynced 539 // domain relevant to a peer 540 // Start - Stop designate a continuous area Keys in an address space 541 // typically the addresses closer to us than to the peer but not closer 542 // another closer peer in between 543 // From - To designates a time interval typically from the last disconnect 544 // till the latest connection (real time traffic is relayed) 545 type DbSyncState struct { 546 Start, Stop Key 547 First, Last uint64 548 } 549 550 // implements the syncer iterator interface 551 // iterates by storage index (~ time of storage = first entry to db) 552 type dbSyncIterator struct { 553 it iterator.Iterator 554 DbSyncState 555 } 556 557 // initialises a sync iterator from a syncToken (passed in with the handshake) 558 func (self *DbStore) NewSyncIterator(state DbSyncState) (si *dbSyncIterator, err error) { 559 if state.First > state.Last { 560 return nil, fmt.Errorf("no entries found") 561 } 562 si = &dbSyncIterator{ 563 it: self.db.NewIterator(), 564 DbSyncState: state, 565 } 566 si.it.Seek(getIndexKey(state.Start)) 567 return si, nil 568 } 569 570 // walk the area from Start to Stop and returns items within time interval 571 // First to Last 572 func (self *dbSyncIterator) Next() (key Key) { 573 for self.it.Valid() { 574 dbkey := self.it.Key() 575 if dbkey[0] != 0 { 576 break 577 } 578 key = Key(make([]byte, len(dbkey)-1)) 579 copy(key[:], dbkey[1:]) 580 if bytes.Compare(key[:], self.Start) <= 0 { 581 self.it.Next() 582 continue 583 } 584 if bytes.Compare(key[:], self.Stop) > 0 { 585 break 586 } 587 var index dpaDBIndex 588 decodeIndex(self.it.Value(), &index) 589 self.it.Next() 590 if (index.Idx >= self.First) && (index.Idx < self.Last) { 591 return 592 } 593 } 594 self.it.Release() 595 return nil 596 }