github.com/jbendotnet/noms@v0.0.0-20190904222105-c43e4293ea92/go/nbs/mem_table.go (about) 1 // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 // Licensed under the Apache License, version 2.0: 3 // http://www.apache.org/licenses/LICENSE-2.0 4 5 package nbs 6 7 import ( 8 "sort" 9 "sync" 10 11 "github.com/attic-labs/noms/go/chunks" 12 "github.com/attic-labs/noms/go/hash" 13 ) 14 15 type memTable struct { 16 chunks map[addr][]byte 17 order []hasRecord // Must maintain the invariant that these are sorted by rec.order 18 maxData, totalData uint64 19 20 snapper snappyEncoder 21 } 22 23 func newMemTable(memTableSize uint64) *memTable { 24 return &memTable{chunks: map[addr][]byte{}, maxData: memTableSize} 25 } 26 27 func (mt *memTable) addChunk(h addr, data []byte) bool { 28 if len(data) == 0 { 29 panic("NBS blocks cannont be zero length") 30 } 31 if _, ok := mt.chunks[h]; ok { 32 return true 33 } 34 dataLen := uint64(len(data)) 35 if mt.totalData+dataLen > mt.maxData { 36 return false 37 } 38 mt.totalData += dataLen 39 mt.chunks[h] = data 40 mt.order = append(mt.order, hasRecord{ 41 &h, 42 h.Prefix(), 43 len(mt.order), 44 false, 45 }) 46 return true 47 } 48 49 func (mt *memTable) count() uint32 { 50 return uint32(len(mt.order)) 51 } 52 53 func (mt *memTable) uncompressedLen() uint64 { 54 return mt.totalData 55 } 56 57 func (mt *memTable) has(h addr) (has bool) { 58 _, has = mt.chunks[h] 59 return 60 } 61 62 func (mt *memTable) hasMany(addrs []hasRecord) (remaining bool) { 63 for i, addr := range addrs { 64 if addr.has { 65 continue 66 } 67 68 if mt.has(*addr.a) { 69 addrs[i].has = true 70 } else { 71 remaining = true 72 } 73 } 74 return 75 } 76 77 func (mt *memTable) get(h addr, stats *Stats) []byte { 78 return mt.chunks[h] 79 } 80 81 func (mt *memTable) getMany(reqs []getRecord, foundChunks chan *chunks.Chunk, wg *sync.WaitGroup, stats *Stats) (remaining bool) { 82 for _, r := range reqs { 83 data := mt.chunks[*r.a] 84 if data != nil { 85 c := chunks.NewChunkWithHash(hash.Hash(*r.a), data) 86 foundChunks <- &c 87 } else { 88 remaining = true 89 } 90 } 91 return 92 } 93 94 func (mt *memTable) extract(chunks chan<- extractRecord) { 95 for _, hrec := range mt.order { 96 chunks <- extractRecord{a: *hrec.a, data: mt.chunks[*hrec.a]} 97 } 98 return 99 } 100 101 func (mt *memTable) write(haver chunkReader, stats *Stats) (name addr, data []byte, count uint32) { 102 maxSize := maxTableSize(uint64(len(mt.order)), mt.totalData) 103 buff := make([]byte, maxSize) 104 tw := newTableWriter(buff, mt.snapper) 105 106 if haver != nil { 107 sort.Sort(hasRecordByPrefix(mt.order)) // hasMany() requires addresses to be sorted. 108 haver.hasMany(mt.order) 109 sort.Sort(hasRecordByOrder(mt.order)) // restore "insertion" order for write 110 } 111 112 for _, addr := range mt.order { 113 if !addr.has { 114 h := addr.a 115 tw.addChunk(*h, mt.chunks[*h]) 116 count++ 117 } 118 } 119 tableSize, name := tw.finish() 120 121 if count > 0 { 122 stats.BytesPerPersist.Sample(uint64(tableSize)) 123 stats.CompressedChunkBytesPerPersist.Sample(uint64(tw.totalCompressedData)) 124 stats.UncompressedChunkBytesPerPersist.Sample(uint64(tw.totalUncompressedData)) 125 stats.ChunksPerPersist.Sample(uint64(count)) 126 } 127 128 return name, buff[:tableSize], count 129 }