github.com/olivere/camlistore@v0.0.0-20140121221811-1b7ac2da0199/third_party/code.google.com/p/leveldb-go/leveldb/memdb/memdb.go (about)

     1  // Copyright 2011 The LevelDB-Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package memdb provides a memory-backed implementation of the db.DB
     6  // interface.
     7  //
     8  // A MemDB's memory consumption increases monotonically, even if keys are
     9  // deleted or values are updated with shorter slices. Callers of the package
    10  // are responsible for explicitly compacting a MemDB into a separate DB
    11  // (whether in-memory or on-disk) when appropriate.
    12  package memdb
    13  
    14  import (
    15  	"encoding/binary"
    16  	"math/rand"
    17  	"sync"
    18  
    19  	"camlistore.org/third_party/code.google.com/p/leveldb-go/leveldb/db"
    20  )
    21  
    22  // maxHeight is the maximum height of a MemDB's skiplist.
    23  const maxHeight = 12
    24  
    25  // A MemDB's skiplist consists of a number of nodes, and each node is
    26  // represented by a variable number of ints: a key-offset, a value-offset, and
    27  // between 1 and maxHeight next nodes. The key-offset and value-offset encode
    28  // the node's key/value pair and are offsets into a MemDB's kvData slice.
    29  // The remaining ints, for the next nodes in the skiplist's linked lists, are
    30  // offsets into a MemDB's nodeData slice.
    31  //
    32  // The fXxx constants represent how to find the Xxx field of a node in the
    33  // nodeData. For example, given an int 30 representing a node, and given
    34  // nodeData[30:36] that looked like [60, 71, 82, 83, 84, 85], then
    35  // nodeData[30 + fKey] = 60 would be the node's key-offset,
    36  // nodeData[30 + fVal] = 71 would be the node's value-offset, and
    37  // nodeData[30 + fNxt + 0] = 82 would be the next node at the height-0 list,
    38  // nodeData[30 + fNxt + 1] = 83 would be the next node at the height-1 list,
    39  // and so on. A node's height is implied by the skiplist construction: a node
    40  // of height x appears in the height-h list iff 0 <= h && h < x.
    41  const (
    42  	fKey = iota
    43  	fVal
    44  	fNxt
    45  )
    46  
    47  const (
    48  	// zeroNode represents the end of a linked list.
    49  	zeroNode = 0
    50  	// headNode represents the start of the linked list. It is equal to -fNxt
    51  	// so that the next nodes at height-h are at nodeData[h].
    52  	// The head node is an artificial node and has no key or value.
    53  	headNode = -fNxt
    54  )
    55  
    56  // A node's key-offset and value-offset fields are offsets into a MemDB's
    57  // kvData slice that stores varint-prefixed strings: the node's key and value.
    58  // A negative offset means a zero-length string, whether explicitly set to
    59  // empty or implicitly set by deletion.
    60  const (
    61  	kvOffsetEmptySlice  = -1
    62  	kvOffsetDeletedNode = -2
    63  )
    64  
    65  // MemDB is a memory-backed implementation of the db.DB interface.
    66  //
    67  // It is safe to call Get, Set, Delete and Find concurrently.
    68  type MemDB struct {
    69  	mutex sync.RWMutex
    70  	// height is the number of such lists, which can increase over time.
    71  	height int
    72  	// cmp defines an ordering on keys.
    73  	cmp db.Comparer
    74  	// kvData is an append-only buffer that holds varint-prefixed strings.
    75  	kvData []byte
    76  	// nodeData is an append-only buffer that holds a node's fields.
    77  	nodeData []int
    78  }
    79  
    80  // MemDB implements the db.DB interface.
    81  var _ db.DB = &MemDB{}
    82  
    83  // load loads a []byte from m.kvData.
    84  func (m *MemDB) load(kvOffset int) (b []byte) {
    85  	if kvOffset < 0 {
    86  		return nil
    87  	}
    88  	bLen, n := binary.Uvarint(m.kvData[kvOffset:])
    89  	return m.kvData[kvOffset+n : kvOffset+n+int(bLen)]
    90  }
    91  
    92  // save saves a []byte to m.kvData.
    93  func (m *MemDB) save(b []byte) (kvOffset int) {
    94  	if len(b) == 0 {
    95  		return kvOffsetEmptySlice
    96  	}
    97  	kvOffset = len(m.kvData)
    98  	var buf [binary.MaxVarintLen64]byte
    99  	length := binary.PutUvarint(buf[:], uint64(len(b)))
   100  	m.kvData = append(m.kvData, buf[:length]...)
   101  	m.kvData = append(m.kvData, b...)
   102  	return kvOffset
   103  }
   104  
   105  // findNode returns the first node n whose key is >= the given key (or nil if
   106  // there is no such node) and whether n's key equals key. The search is based
   107  // solely on the contents of a node's key. Whether or not that key was
   108  // previously deleted from the MemDB is not relevant.
   109  //
   110  // If prev is non-nil, it also sets the first m.height elements of prev to the
   111  // preceding node at each height.
   112  func (m *MemDB) findNode(key []byte, prev *[maxHeight]int) (n int, exactMatch bool) {
   113  	for h, p := m.height-1, headNode; h >= 0; h-- {
   114  		// Walk the skiplist at height h until we find either a zero node
   115  		// or one whose key is >= the given key.
   116  		n = m.nodeData[p+fNxt+h]
   117  		for {
   118  			if n == zeroNode {
   119  				exactMatch = false
   120  				break
   121  			}
   122  			kOff := m.nodeData[n+fKey]
   123  			if c := m.cmp.Compare(m.load(kOff), key); c >= 0 {
   124  				exactMatch = c == 0
   125  				break
   126  			}
   127  			p, n = n, m.nodeData[n+fNxt+h]
   128  		}
   129  		if prev != nil {
   130  			(*prev)[h] = p
   131  		}
   132  	}
   133  	return n, exactMatch
   134  }
   135  
   136  // Get implements DB.Get, as documented in the leveldb/db package.
   137  func (m *MemDB) Get(key []byte, o *db.ReadOptions) (value []byte, err error) {
   138  	m.mutex.RLock()
   139  	defer m.mutex.RUnlock()
   140  	n, exactMatch := m.findNode(key, nil)
   141  	vOff := m.nodeData[n+fVal]
   142  	if !exactMatch || vOff == kvOffsetDeletedNode {
   143  		return nil, db.ErrNotFound
   144  	}
   145  	return m.load(vOff), nil
   146  }
   147  
   148  // Set implements DB.Set, as documented in the leveldb/db package.
   149  func (m *MemDB) Set(key, value []byte, o *db.WriteOptions) error {
   150  	m.mutex.Lock()
   151  	defer m.mutex.Unlock()
   152  	// Find the node, and its predecessors at all heights.
   153  	var prev [maxHeight]int
   154  	n, exactMatch := m.findNode(key, &prev)
   155  	if exactMatch {
   156  		m.nodeData[n+fVal] = m.save(value)
   157  		return nil
   158  	}
   159  	// Choose the new node's height, branching with 25% probability.
   160  	h := 1
   161  	for h < maxHeight && rand.Intn(4) == 0 {
   162  		h++
   163  	}
   164  	// Raise the skiplist's height to the node's height, if necessary.
   165  	if m.height < h {
   166  		for i := m.height; i < h; i++ {
   167  			prev[i] = headNode
   168  		}
   169  		m.height = h
   170  	}
   171  	// Insert the new node.
   172  	var x [fNxt + maxHeight]int
   173  	n1 := len(m.nodeData)
   174  	x[fKey] = m.save(key)
   175  	x[fVal] = m.save(value)
   176  	for i := 0; i < h; i++ {
   177  		j := prev[i] + fNxt + i
   178  		x[fNxt+i] = m.nodeData[j]
   179  		m.nodeData[j] = n1
   180  	}
   181  	m.nodeData = append(m.nodeData, x[:fNxt+h]...)
   182  	return nil
   183  }
   184  
   185  // Delete implements DB.Delete, as documented in the leveldb/db package.
   186  func (m *MemDB) Delete(key []byte, o *db.WriteOptions) error {
   187  	m.mutex.Lock()
   188  	defer m.mutex.Unlock()
   189  	n, exactMatch := m.findNode(key, nil)
   190  	if !exactMatch || m.nodeData[n+fVal] == kvOffsetDeletedNode {
   191  		return db.ErrNotFound
   192  	}
   193  	m.nodeData[n+fVal] = kvOffsetDeletedNode
   194  	return nil
   195  }
   196  
   197  // Find implements DB.Find, as documented in the leveldb/db package.
   198  func (m *MemDB) Find(key []byte, o *db.ReadOptions) db.Iterator {
   199  	m.mutex.RLock()
   200  	defer m.mutex.RUnlock()
   201  	n, _ := m.findNode(key, nil)
   202  	for n != zeroNode && m.nodeData[n+fVal] == kvOffsetDeletedNode {
   203  		n = m.nodeData[n+fNxt]
   204  	}
   205  	t := &iterator{
   206  		m:           m,
   207  		restartNode: n,
   208  	}
   209  	t.fill()
   210  	// The iterator is positioned at the first node >= key. The iterator API
   211  	// requires that the caller the Next first, so we set t.i0 to -1.
   212  	t.i0 = -1
   213  	return t
   214  }
   215  
   216  // Close implements DB.Close, as documented in the leveldb/db package.
   217  func (m *MemDB) Close() error {
   218  	return nil
   219  }
   220  
   221  // ApproximateMemoryUsage returns the approximate memory usage of the MemDB.
   222  func (m *MemDB) ApproximateMemoryUsage() int {
   223  	m.mutex.RLock()
   224  	defer m.mutex.RUnlock()
   225  	return len(m.kvData)
   226  }
   227  
   228  // New returns a new MemDB.
   229  func New(o *db.Options) *MemDB {
   230  	return &MemDB{
   231  		height: 1,
   232  		cmp:    o.GetComparer(),
   233  		kvData: make([]byte, 0, 4096),
   234  		// The first maxHeight values of nodeData are the next nodes after the
   235  		// head node at each possible height. Their initial value is zeroNode.
   236  		nodeData: make([]int, maxHeight, 256),
   237  	}
   238  }
   239  
   240  // iterator is a MemDB iterator that buffers upcoming results, so that it does
   241  // not have to acquire the MemDB's mutex on each Next call.
   242  type iterator struct {
   243  	m *MemDB
   244  	// restartNode is the node to start refilling the buffer from.
   245  	restartNode int
   246  	// i0 is the current iterator position with respect to buf. A value of -1
   247  	// means that the iterator is at the start, end or both of the iteration.
   248  	// i1 is the number of buffered entries.
   249  	// Invariant: -1 <= i0 && i0 < i1 && i1 <= len(buf).
   250  	i0, i1 int
   251  	// buf buffers up to 32 key/value pairs.
   252  	buf [32][2][]byte
   253  }
   254  
   255  // iterator implements the db.Iterator interface.
   256  var _ db.Iterator = &iterator{}
   257  
   258  // fill fills the iterator's buffer with key/value pairs from the MemDB.
   259  //
   260  // Precondition: t.m.mutex is locked for reading.
   261  func (t *iterator) fill() {
   262  	i, n := 0, t.restartNode
   263  	for i < len(t.buf) && n != zeroNode {
   264  		if t.m.nodeData[n+fVal] != kvOffsetDeletedNode {
   265  			t.buf[i][fKey] = t.m.load(t.m.nodeData[n+fKey])
   266  			t.buf[i][fVal] = t.m.load(t.m.nodeData[n+fVal])
   267  			i++
   268  		}
   269  		n = t.m.nodeData[n+fNxt]
   270  	}
   271  	if i == 0 {
   272  		// There were no non-deleted nodes on or after t.restartNode.
   273  		// The iterator is exhausted.
   274  		t.i0 = -1
   275  	} else {
   276  		t.i0 = 0
   277  	}
   278  	t.i1 = i
   279  	t.restartNode = n
   280  }
   281  
   282  // Next implements Iterator.Next, as documented in the leveldb/db package.
   283  func (t *iterator) Next() bool {
   284  	t.i0++
   285  	if t.i0 < t.i1 {
   286  		return true
   287  	}
   288  	if t.restartNode == zeroNode {
   289  		t.i0 = -1
   290  		t.i1 = 0
   291  		return false
   292  	}
   293  	t.m.mutex.RLock()
   294  	defer t.m.mutex.RUnlock()
   295  	t.fill()
   296  	return true
   297  }
   298  
   299  // Key implements Iterator.Key, as documented in the leveldb/db package.
   300  func (t *iterator) Key() []byte {
   301  	if t.i0 < 0 {
   302  		return nil
   303  	}
   304  	return t.buf[t.i0][fKey]
   305  }
   306  
   307  // Value implements Iterator.Value, as documented in the leveldb/db package.
   308  func (t *iterator) Value() []byte {
   309  	if t.i0 < 0 {
   310  		return nil
   311  	}
   312  	return t.buf[t.i0][fVal]
   313  }
   314  
   315  // Close implements Iterator.Close, as documented in the leveldb/db package.
   316  func (t *iterator) Close() error {
   317  	return nil
   318  }