github.com/olivere/camlistore@v0.0.0-20140121221811-1b7ac2da0199/third_party/code.google.com/p/leveldb-go/leveldb/memdb/memdb.go (about) 1 // Copyright 2011 The LevelDB-Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Package memdb provides a memory-backed implementation of the db.DB 6 // interface. 7 // 8 // A MemDB's memory consumption increases monotonically, even if keys are 9 // deleted or values are updated with shorter slices. Callers of the package 10 // are responsible for explicitly compacting a MemDB into a separate DB 11 // (whether in-memory or on-disk) when appropriate. 12 package memdb 13 14 import ( 15 "encoding/binary" 16 "math/rand" 17 "sync" 18 19 "camlistore.org/third_party/code.google.com/p/leveldb-go/leveldb/db" 20 ) 21 22 // maxHeight is the maximum height of a MemDB's skiplist. 23 const maxHeight = 12 24 25 // A MemDB's skiplist consists of a number of nodes, and each node is 26 // represented by a variable number of ints: a key-offset, a value-offset, and 27 // between 1 and maxHeight next nodes. The key-offset and value-offset encode 28 // the node's key/value pair and are offsets into a MemDB's kvData slice. 29 // The remaining ints, for the next nodes in the skiplist's linked lists, are 30 // offsets into a MemDB's nodeData slice. 31 // 32 // The fXxx constants represent how to find the Xxx field of a node in the 33 // nodeData. For example, given an int 30 representing a node, and given 34 // nodeData[30:36] that looked like [60, 71, 82, 83, 84, 85], then 35 // nodeData[30 + fKey] = 60 would be the node's key-offset, 36 // nodeData[30 + fVal] = 71 would be the node's value-offset, and 37 // nodeData[30 + fNxt + 0] = 82 would be the next node at the height-0 list, 38 // nodeData[30 + fNxt + 1] = 83 would be the next node at the height-1 list, 39 // and so on. A node's height is implied by the skiplist construction: a node 40 // of height x appears in the height-h list iff 0 <= h && h < x. 41 const ( 42 fKey = iota 43 fVal 44 fNxt 45 ) 46 47 const ( 48 // zeroNode represents the end of a linked list. 49 zeroNode = 0 50 // headNode represents the start of the linked list. It is equal to -fNxt 51 // so that the next nodes at height-h are at nodeData[h]. 52 // The head node is an artificial node and has no key or value. 53 headNode = -fNxt 54 ) 55 56 // A node's key-offset and value-offset fields are offsets into a MemDB's 57 // kvData slice that stores varint-prefixed strings: the node's key and value. 58 // A negative offset means a zero-length string, whether explicitly set to 59 // empty or implicitly set by deletion. 60 const ( 61 kvOffsetEmptySlice = -1 62 kvOffsetDeletedNode = -2 63 ) 64 65 // MemDB is a memory-backed implementation of the db.DB interface. 66 // 67 // It is safe to call Get, Set, Delete and Find concurrently. 68 type MemDB struct { 69 mutex sync.RWMutex 70 // height is the number of such lists, which can increase over time. 71 height int 72 // cmp defines an ordering on keys. 73 cmp db.Comparer 74 // kvData is an append-only buffer that holds varint-prefixed strings. 75 kvData []byte 76 // nodeData is an append-only buffer that holds a node's fields. 77 nodeData []int 78 } 79 80 // MemDB implements the db.DB interface. 81 var _ db.DB = &MemDB{} 82 83 // load loads a []byte from m.kvData. 84 func (m *MemDB) load(kvOffset int) (b []byte) { 85 if kvOffset < 0 { 86 return nil 87 } 88 bLen, n := binary.Uvarint(m.kvData[kvOffset:]) 89 return m.kvData[kvOffset+n : kvOffset+n+int(bLen)] 90 } 91 92 // save saves a []byte to m.kvData. 93 func (m *MemDB) save(b []byte) (kvOffset int) { 94 if len(b) == 0 { 95 return kvOffsetEmptySlice 96 } 97 kvOffset = len(m.kvData) 98 var buf [binary.MaxVarintLen64]byte 99 length := binary.PutUvarint(buf[:], uint64(len(b))) 100 m.kvData = append(m.kvData, buf[:length]...) 101 m.kvData = append(m.kvData, b...) 102 return kvOffset 103 } 104 105 // findNode returns the first node n whose key is >= the given key (or nil if 106 // there is no such node) and whether n's key equals key. The search is based 107 // solely on the contents of a node's key. Whether or not that key was 108 // previously deleted from the MemDB is not relevant. 109 // 110 // If prev is non-nil, it also sets the first m.height elements of prev to the 111 // preceding node at each height. 112 func (m *MemDB) findNode(key []byte, prev *[maxHeight]int) (n int, exactMatch bool) { 113 for h, p := m.height-1, headNode; h >= 0; h-- { 114 // Walk the skiplist at height h until we find either a zero node 115 // or one whose key is >= the given key. 116 n = m.nodeData[p+fNxt+h] 117 for { 118 if n == zeroNode { 119 exactMatch = false 120 break 121 } 122 kOff := m.nodeData[n+fKey] 123 if c := m.cmp.Compare(m.load(kOff), key); c >= 0 { 124 exactMatch = c == 0 125 break 126 } 127 p, n = n, m.nodeData[n+fNxt+h] 128 } 129 if prev != nil { 130 (*prev)[h] = p 131 } 132 } 133 return n, exactMatch 134 } 135 136 // Get implements DB.Get, as documented in the leveldb/db package. 137 func (m *MemDB) Get(key []byte, o *db.ReadOptions) (value []byte, err error) { 138 m.mutex.RLock() 139 defer m.mutex.RUnlock() 140 n, exactMatch := m.findNode(key, nil) 141 vOff := m.nodeData[n+fVal] 142 if !exactMatch || vOff == kvOffsetDeletedNode { 143 return nil, db.ErrNotFound 144 } 145 return m.load(vOff), nil 146 } 147 148 // Set implements DB.Set, as documented in the leveldb/db package. 149 func (m *MemDB) Set(key, value []byte, o *db.WriteOptions) error { 150 m.mutex.Lock() 151 defer m.mutex.Unlock() 152 // Find the node, and its predecessors at all heights. 153 var prev [maxHeight]int 154 n, exactMatch := m.findNode(key, &prev) 155 if exactMatch { 156 m.nodeData[n+fVal] = m.save(value) 157 return nil 158 } 159 // Choose the new node's height, branching with 25% probability. 160 h := 1 161 for h < maxHeight && rand.Intn(4) == 0 { 162 h++ 163 } 164 // Raise the skiplist's height to the node's height, if necessary. 165 if m.height < h { 166 for i := m.height; i < h; i++ { 167 prev[i] = headNode 168 } 169 m.height = h 170 } 171 // Insert the new node. 172 var x [fNxt + maxHeight]int 173 n1 := len(m.nodeData) 174 x[fKey] = m.save(key) 175 x[fVal] = m.save(value) 176 for i := 0; i < h; i++ { 177 j := prev[i] + fNxt + i 178 x[fNxt+i] = m.nodeData[j] 179 m.nodeData[j] = n1 180 } 181 m.nodeData = append(m.nodeData, x[:fNxt+h]...) 182 return nil 183 } 184 185 // Delete implements DB.Delete, as documented in the leveldb/db package. 186 func (m *MemDB) Delete(key []byte, o *db.WriteOptions) error { 187 m.mutex.Lock() 188 defer m.mutex.Unlock() 189 n, exactMatch := m.findNode(key, nil) 190 if !exactMatch || m.nodeData[n+fVal] == kvOffsetDeletedNode { 191 return db.ErrNotFound 192 } 193 m.nodeData[n+fVal] = kvOffsetDeletedNode 194 return nil 195 } 196 197 // Find implements DB.Find, as documented in the leveldb/db package. 198 func (m *MemDB) Find(key []byte, o *db.ReadOptions) db.Iterator { 199 m.mutex.RLock() 200 defer m.mutex.RUnlock() 201 n, _ := m.findNode(key, nil) 202 for n != zeroNode && m.nodeData[n+fVal] == kvOffsetDeletedNode { 203 n = m.nodeData[n+fNxt] 204 } 205 t := &iterator{ 206 m: m, 207 restartNode: n, 208 } 209 t.fill() 210 // The iterator is positioned at the first node >= key. The iterator API 211 // requires that the caller the Next first, so we set t.i0 to -1. 212 t.i0 = -1 213 return t 214 } 215 216 // Close implements DB.Close, as documented in the leveldb/db package. 217 func (m *MemDB) Close() error { 218 return nil 219 } 220 221 // ApproximateMemoryUsage returns the approximate memory usage of the MemDB. 222 func (m *MemDB) ApproximateMemoryUsage() int { 223 m.mutex.RLock() 224 defer m.mutex.RUnlock() 225 return len(m.kvData) 226 } 227 228 // New returns a new MemDB. 229 func New(o *db.Options) *MemDB { 230 return &MemDB{ 231 height: 1, 232 cmp: o.GetComparer(), 233 kvData: make([]byte, 0, 4096), 234 // The first maxHeight values of nodeData are the next nodes after the 235 // head node at each possible height. Their initial value is zeroNode. 236 nodeData: make([]int, maxHeight, 256), 237 } 238 } 239 240 // iterator is a MemDB iterator that buffers upcoming results, so that it does 241 // not have to acquire the MemDB's mutex on each Next call. 242 type iterator struct { 243 m *MemDB 244 // restartNode is the node to start refilling the buffer from. 245 restartNode int 246 // i0 is the current iterator position with respect to buf. A value of -1 247 // means that the iterator is at the start, end or both of the iteration. 248 // i1 is the number of buffered entries. 249 // Invariant: -1 <= i0 && i0 < i1 && i1 <= len(buf). 250 i0, i1 int 251 // buf buffers up to 32 key/value pairs. 252 buf [32][2][]byte 253 } 254 255 // iterator implements the db.Iterator interface. 256 var _ db.Iterator = &iterator{} 257 258 // fill fills the iterator's buffer with key/value pairs from the MemDB. 259 // 260 // Precondition: t.m.mutex is locked for reading. 261 func (t *iterator) fill() { 262 i, n := 0, t.restartNode 263 for i < len(t.buf) && n != zeroNode { 264 if t.m.nodeData[n+fVal] != kvOffsetDeletedNode { 265 t.buf[i][fKey] = t.m.load(t.m.nodeData[n+fKey]) 266 t.buf[i][fVal] = t.m.load(t.m.nodeData[n+fVal]) 267 i++ 268 } 269 n = t.m.nodeData[n+fNxt] 270 } 271 if i == 0 { 272 // There were no non-deleted nodes on or after t.restartNode. 273 // The iterator is exhausted. 274 t.i0 = -1 275 } else { 276 t.i0 = 0 277 } 278 t.i1 = i 279 t.restartNode = n 280 } 281 282 // Next implements Iterator.Next, as documented in the leveldb/db package. 283 func (t *iterator) Next() bool { 284 t.i0++ 285 if t.i0 < t.i1 { 286 return true 287 } 288 if t.restartNode == zeroNode { 289 t.i0 = -1 290 t.i1 = 0 291 return false 292 } 293 t.m.mutex.RLock() 294 defer t.m.mutex.RUnlock() 295 t.fill() 296 return true 297 } 298 299 // Key implements Iterator.Key, as documented in the leveldb/db package. 300 func (t *iterator) Key() []byte { 301 if t.i0 < 0 { 302 return nil 303 } 304 return t.buf[t.i0][fKey] 305 } 306 307 // Value implements Iterator.Value, as documented in the leveldb/db package. 308 func (t *iterator) Value() []byte { 309 if t.i0 < 0 { 310 return nil 311 } 312 return t.buf[t.i0][fVal] 313 } 314 315 // Close implements Iterator.Close, as documented in the leveldb/db package. 316 func (t *iterator) Close() error { 317 return nil 318 }