github.com/jbendotnet/noms@v0.0.0-20190904222105-c43e4293ea92/go/nbs/mem_table.go (about)

     1  // Copyright 2016 Attic Labs, Inc. All rights reserved.
     2  // Licensed under the Apache License, version 2.0:
     3  // http://www.apache.org/licenses/LICENSE-2.0
     4  
     5  package nbs
     6  
     7  import (
     8  	"sort"
     9  	"sync"
    10  
    11  	"github.com/attic-labs/noms/go/chunks"
    12  	"github.com/attic-labs/noms/go/hash"
    13  )
    14  
    15  type memTable struct {
    16  	chunks             map[addr][]byte
    17  	order              []hasRecord // Must maintain the invariant that these are sorted by rec.order
    18  	maxData, totalData uint64
    19  
    20  	snapper snappyEncoder
    21  }
    22  
    23  func newMemTable(memTableSize uint64) *memTable {
    24  	return &memTable{chunks: map[addr][]byte{}, maxData: memTableSize}
    25  }
    26  
    27  func (mt *memTable) addChunk(h addr, data []byte) bool {
    28  	if len(data) == 0 {
    29  		panic("NBS blocks cannont be zero length")
    30  	}
    31  	if _, ok := mt.chunks[h]; ok {
    32  		return true
    33  	}
    34  	dataLen := uint64(len(data))
    35  	if mt.totalData+dataLen > mt.maxData {
    36  		return false
    37  	}
    38  	mt.totalData += dataLen
    39  	mt.chunks[h] = data
    40  	mt.order = append(mt.order, hasRecord{
    41  		&h,
    42  		h.Prefix(),
    43  		len(mt.order),
    44  		false,
    45  	})
    46  	return true
    47  }
    48  
    49  func (mt *memTable) count() uint32 {
    50  	return uint32(len(mt.order))
    51  }
    52  
    53  func (mt *memTable) uncompressedLen() uint64 {
    54  	return mt.totalData
    55  }
    56  
    57  func (mt *memTable) has(h addr) (has bool) {
    58  	_, has = mt.chunks[h]
    59  	return
    60  }
    61  
    62  func (mt *memTable) hasMany(addrs []hasRecord) (remaining bool) {
    63  	for i, addr := range addrs {
    64  		if addr.has {
    65  			continue
    66  		}
    67  
    68  		if mt.has(*addr.a) {
    69  			addrs[i].has = true
    70  		} else {
    71  			remaining = true
    72  		}
    73  	}
    74  	return
    75  }
    76  
    77  func (mt *memTable) get(h addr, stats *Stats) []byte {
    78  	return mt.chunks[h]
    79  }
    80  
    81  func (mt *memTable) getMany(reqs []getRecord, foundChunks chan *chunks.Chunk, wg *sync.WaitGroup, stats *Stats) (remaining bool) {
    82  	for _, r := range reqs {
    83  		data := mt.chunks[*r.a]
    84  		if data != nil {
    85  			c := chunks.NewChunkWithHash(hash.Hash(*r.a), data)
    86  			foundChunks <- &c
    87  		} else {
    88  			remaining = true
    89  		}
    90  	}
    91  	return
    92  }
    93  
    94  func (mt *memTable) extract(chunks chan<- extractRecord) {
    95  	for _, hrec := range mt.order {
    96  		chunks <- extractRecord{a: *hrec.a, data: mt.chunks[*hrec.a]}
    97  	}
    98  	return
    99  }
   100  
   101  func (mt *memTable) write(haver chunkReader, stats *Stats) (name addr, data []byte, count uint32) {
   102  	maxSize := maxTableSize(uint64(len(mt.order)), mt.totalData)
   103  	buff := make([]byte, maxSize)
   104  	tw := newTableWriter(buff, mt.snapper)
   105  
   106  	if haver != nil {
   107  		sort.Sort(hasRecordByPrefix(mt.order)) // hasMany() requires addresses to be sorted.
   108  		haver.hasMany(mt.order)
   109  		sort.Sort(hasRecordByOrder(mt.order)) // restore "insertion" order for write
   110  	}
   111  
   112  	for _, addr := range mt.order {
   113  		if !addr.has {
   114  			h := addr.a
   115  			tw.addChunk(*h, mt.chunks[*h])
   116  			count++
   117  		}
   118  	}
   119  	tableSize, name := tw.finish()
   120  
   121  	if count > 0 {
   122  		stats.BytesPerPersist.Sample(uint64(tableSize))
   123  		stats.CompressedChunkBytesPerPersist.Sample(uint64(tw.totalCompressedData))
   124  		stats.UncompressedChunkBytesPerPersist.Sample(uint64(tw.totalUncompressedData))
   125  		stats.ChunksPerPersist.Sample(uint64(count))
   126  	}
   127  
   128  	return name, buff[:tableSize], count
   129  }