github.com/fengyoulin/shm@v0.0.0-20200305015033-287e184bdf0a/map.go (about) 1 package shm 2 3 import ( 4 "errors" 5 "github.com/fengyoulin/shm/database" 6 "github.com/fengyoulin/shm/mapping" 7 "hash/crc32" 8 "reflect" 9 "sync/atomic" 10 "time" 11 "unsafe" 12 ) 13 14 // Map is a shared map 15 type Map struct { 16 mp *mapping.Mapping 17 head *header 18 hash *[maxMapCap]hash 19 data uintptr 20 try int 21 } 22 23 // header in database 24 type header struct { 25 len int32 26 cap int32 27 keySize int32 28 bucketSize int32 29 hashOff uint32 30 dataOff uint32 31 next int32 32 deleteLink int32 33 _ [8]int32 34 } 35 36 // hash as [4]int32 37 // 1st for index 38 // 2nd for serial 39 // 3rd for lock 40 // 4th for chain len, debug purpose 41 type hash [4]int32 42 43 // bucket header 44 type bucket struct { 45 next int32 46 hash int32 47 used int32 48 _ int32 49 // key [keySize]byte 50 // value [bucketSize]byte 51 } 52 53 const ( 54 maxMapCap = 64 * 1024 * 1024 55 minKeySize = 8 56 maxKeySize = 256 57 maxBktSize = 4096 58 ) 59 60 var ( 61 // ErrMapCap on param validate 62 ErrMapCap = errors.New("map cap too large or too small") 63 // ErrKeyLen on param validate 64 ErrKeyLen = errors.New("key too long or too short") 65 // ErrValLen on param validate 66 ErrValLen = errors.New("value too large or too small") 67 // ErrKeyNot on get and not add 68 ErrKeyNot = errors.New("key not found in map") 69 // ErrDbSize on open an exist db 70 ErrDbSize = errors.New("database size mismatch") 71 // ErrDbFull on add a new key 72 ErrDbFull = errors.New("no more space in map") 73 // ErrTryEnd on add or delete 74 ErrTryEnd = errors.New("cannot add after too many tries") 75 ) 76 77 // Create or open a shared map database 78 func Create(path string, mapCap, keyLen, valueLen, maxTry int, wait time.Duration) (m *Map, err error) { 79 var hdr header 80 if maxTry <= 0 { 81 maxTry = 20 82 } 83 if mapCap <= 0 || mapCap > maxMapCap { 84 err = ErrMapCap 85 return 86 } 87 // round up to power of 2 88 mapCap-- 89 mapCap |= mapCap >> 1 90 mapCap |= mapCap >> 2 91 mapCap |= mapCap >> 4 92 mapCap |= mapCap >> 8 93 mapCap++ 94 if mapCap < 8 { 95 mapCap = 8 96 } 97 hdr.cap = int32(mapCap) 98 if keyLen < minKeySize-1 || keyLen > maxKeySize-1 { 99 err = ErrKeyLen 100 return 101 } 102 // plus one byte for length 103 keyLen = (keyLen + 1 + 3) & (^3) 104 hdr.keySize = int32(keyLen) 105 if valueLen < 0 || valueLen > maxBktSize-int(unsafe.Sizeof(bucket{}))-keyLen { 106 err = ErrValLen 107 return 108 } 109 bktLen := int(unsafe.Sizeof(bucket{})) + keyLen + valueLen 110 // round up to multiples of 16 111 bktLen = (bktLen + 15) & (^15) 112 hdr.bucketSize = int32(bktLen) 113 // hash area after header 114 hdr.hashOff = uint32(unsafe.Sizeof(hdr)) 115 // hash area size 116 hashSize := int(unsafe.Sizeof(hash{})) * mapCap 117 hdr.dataOff = hdr.hashOff + uint32(hashSize) 118 // total size, header + hash + buckets 119 size := int(hdr.dataOff) + int(hdr.cap*hdr.bucketSize) 120 mp, ul, err := database.Open(path, size, wait) 121 if err != nil { 122 return 123 } 124 defer func() { 125 // close db if unlock failed 126 if e := ul(); e != nil && err == nil { 127 err = e 128 _ = m.Close() 129 } 130 }() 131 m = &Map{ 132 mp: mp, 133 try: maxTry, 134 } 135 err = m.init(&hdr) 136 // close db if init failed 137 if err != nil { 138 _ = m.Close() 139 return 140 } 141 return 142 } 143 144 // Close the shared map database 145 func (m *Map) Close() error { 146 err := m.mp.Close() 147 m.mp = nil 148 m.head = nil 149 m.hash = nil 150 m.data = 0 151 return err 152 } 153 154 // Get or add an key 155 // return the value in a byte slice on success 156 // return error on failure if !add, maybe because of: 157 // too many tries on a highly parallel situation, or 158 // no more space in the database, or 159 // hash func failed 160 func (m *Map) Get(key string, add bool) (b []byte, err error) { 161 h, err := hashFunc(key) 162 if err != nil { 163 return 164 } 165 ptr := m.hashPtr(h) 166 try := m.try 167 var newIdx int32 168 var target *bucket 169 defer func() { 170 if target != nil { 171 m.free(newIdx) 172 } 173 }() 174 var lastCheck bool 175 for try > 0 { 176 try-- 177 index := ptr.index() 178 serial := ptr.serial() 179 // traverse the bucket chain 180 for idx := index; idx >= 0; { 181 bkt := m.bucket(idx) 182 if key != bkt.key() { 183 idx = bkt.next 184 continue 185 } 186 b = bkt.value(m) 187 return 188 } 189 // last check on no space 190 if lastCheck { 191 err = ErrDbFull 192 return 193 } 194 // not found 195 if !add { 196 err = ErrKeyNot 197 return 198 } 199 if target == nil { 200 newIdx = m.alloc() 201 if newIdx < 0 { 202 // maybe just added by some other, do last check 203 if ptr.serial() != serial { 204 lastCheck = true 205 continue 206 } 207 err = ErrDbFull 208 return 209 } 210 target = m.bucket(newIdx) 211 target.setKey(m, key) 212 target.hash = h 213 } 214 // lock succeed if serial not changed 215 if ptr.lock(serial) { 216 target.next = index 217 ptr.setIndex(newIdx) 218 target.used = 1 219 ptr.addLength(1) 220 ptr.unlock() 221 atomic.AddInt32(&m.head.len, 1) 222 b = target.value(m) 223 target = nil 224 return 225 } 226 } 227 return nil, ErrTryEnd 228 } 229 230 // Delete a key 231 // return false on failure, maybe because of: 232 // too many tries on a highly parallel situation, or 233 // hash func failed 234 func (m *Map) Delete(key string) bool { 235 h, err := hashFunc(key) 236 if err != nil { 237 return false 238 } 239 ptr := m.hashPtr(h) 240 try := m.try 241 for try > 0 { 242 try-- 243 index := ptr.index() 244 serial := ptr.serial() 245 var last, target *bucket 246 // traverse the bucket chain 247 idx := index 248 for idx >= 0 { 249 bkt := m.bucket(idx) 250 if key != bkt.key() { 251 last = bkt 252 idx = bkt.next 253 continue 254 } 255 target = bkt 256 break 257 } 258 // not found 259 if target == nil { 260 return true 261 } 262 // lock succeed if serial not changed 263 if ptr.lock(serial) { 264 target.used = 0 265 if last != nil { 266 last.next = target.next 267 } else { 268 ptr.setIndex(target.next) 269 } 270 ptr.addLength(-1) 271 ptr.unlock() 272 atomic.AddInt32(&m.head.len, -1) 273 m.free(idx) 274 return true 275 } 276 } 277 return false 278 } 279 280 // Foreach key/value pair in the map call fn 281 // stop on fn return false or finished 282 func (m *Map) Foreach(fn func(key string, value []byte) bool) { 283 for i := int32(0); i < m.head.cap; i++ { 284 bkt := m.bucket(i) 285 if bkt.used == 0 { 286 continue 287 } 288 if !fn(bkt.key(), bkt.value(m)) { 289 return 290 } 291 } 292 } 293 294 // Cap return map capacity, cannot grow 295 func (m *Map) Cap() int { 296 return int(m.head.cap) 297 } 298 299 // Len return item count in map 300 func (m *Map) Len() int { 301 return int(atomic.LoadInt32(&m.head.len)) 302 } 303 304 // from a exist db, or a newly created one 305 func (m *Map) init(h *header) error { 306 data := m.mp.Bytes() 307 sh := (*reflect.SliceHeader)(unsafe.Pointer(&data)) 308 head := (*header)(unsafe.Pointer(sh.Data)) 309 if head.cap != 0 { 310 // this branch opened a exist db 311 if head.cap != h.cap || 312 head.keySize != h.keySize || 313 head.bucketSize != h.bucketSize || 314 head.hashOff != h.hashOff || 315 head.dataOff != h.dataOff { 316 return ErrDbSize 317 } 318 } else { 319 // new db, init hash area, set index to -1 320 hs := (*[maxMapCap]hash)(unsafe.Pointer(sh.Data + uintptr(h.hashOff))) 321 for i := 0; i < int(h.cap); i++ { 322 (*hs)[i][0] = -1 323 } 324 // set deleted link to -1 325 head.deleteLink = -1 326 // copy header params 327 head.keySize = h.keySize 328 head.bucketSize = h.bucketSize 329 head.hashOff = h.hashOff 330 head.dataOff = h.dataOff 331 // set cap at the end 332 head.cap = h.cap 333 } 334 m.head = head 335 m.hash = (*[maxMapCap]hash)(unsafe.Pointer(sh.Data + uintptr(head.hashOff))) 336 m.data = sh.Data + uintptr(head.dataOff) 337 return nil 338 } 339 340 // bucket index 341 func (m *Map) alloc() int32 { 342 // from deleted first 343 for { 344 del := m.head.deleteLink 345 if del < 0 { 346 break 347 } 348 bkt := m.bucket(del) 349 if atomic.CompareAndSwapInt32(&m.head.deleteLink, del, bkt.next) { 350 bkt.next = -1 351 return del 352 } 353 } 354 // from "next" second 355 for { 356 next := m.head.next 357 if next >= m.head.cap { 358 break 359 } 360 if atomic.CompareAndSwapInt32(&m.head.next, next, next+1) { 361 m.bucket(next).next = -1 362 return next 363 } 364 } 365 return -1 366 } 367 368 // bucket index 369 func (m *Map) free(i int32) { 370 bkt := m.bucket(i) 371 // put to deleted link 372 for { 373 last := m.head.deleteLink 374 bkt.next = last 375 if atomic.CompareAndSwapInt32(&m.head.deleteLink, last, i) { 376 return 377 } 378 } 379 } 380 381 // index to pointer 382 func (m *Map) bucket(i int32) *bucket { 383 return (*bucket)(unsafe.Pointer(uintptr(unsafe.Pointer(m.head)) + uintptr(m.head.dataOff+uint32(m.head.bucketSize*i)))) 384 } 385 386 // pointer to index 387 func (m *Map) index(b *bucket) int32 { 388 return int32(uint32(uintptr(unsafe.Pointer(b))-uintptr(unsafe.Pointer(m.head)))-m.head.dataOff) / m.head.bucketSize 389 } 390 391 // hash pointer 392 func (m *Map) hashPtr(h int32) *hash { 393 return &(*m.hash)[int(uint(h)%uint(m.head.cap))] 394 } 395 396 // the first bucket's index in chain 397 func (h *hash) index() int32 { 398 return (*h)[0] 399 } 400 401 // serial number for change check 402 func (h *hash) serial() int32 { 403 return (*h)[1] 404 } 405 406 // set the first bucket index of chain 407 func (h *hash) setIndex(index int32) { 408 (*h)[0] = index 409 } 410 411 // lock the bucket chain 412 func (h *hash) lock(serial int32) bool { 413 if atomic.CompareAndSwapInt32(&(*h)[2], 0, 1) { 414 if serial == (*h)[1] { 415 return true 416 } 417 atomic.StoreInt32(&(*h)[2], 0) 418 } 419 return false 420 } 421 422 // unlock the bucket chain 423 func (h *hash) unlock() { 424 (*h)[1]++ 425 atomic.StoreInt32(&(*h)[2], 0) 426 } 427 428 // chain length 429 func (h *hash) length() int { 430 return int((*h)[3]) 431 } 432 433 // add delta to chain length 434 func (h *hash) addLength(delta int) { 435 (*h)[3] += int32(delta) 436 } 437 438 // find chain 439 func (b *bucket) hashPtr(m *Map) *hash { 440 return m.hashPtr(b.hash) 441 } 442 443 // bucket key 444 func (b *bucket) key() (s string) { 445 a := uintptr(unsafe.Pointer(b)) + unsafe.Sizeof(bucket{}) 446 h := (*reflect.StringHeader)(unsafe.Pointer(&s)) 447 h.Data = a + 1 448 h.Len = int(*(*uint8)(unsafe.Pointer(a))) 449 return 450 } 451 452 // bucket value 453 func (b *bucket) value(m *Map) (d []byte) { 454 a := uintptr(unsafe.Pointer(b)) + unsafe.Sizeof(bucket{}) + uintptr(m.head.keySize) 455 h := (*reflect.SliceHeader)(unsafe.Pointer(&d)) 456 h.Data = a 457 h.Cap = int(m.head.bucketSize) - int(unsafe.Sizeof(bucket{})) - int(m.head.keySize) 458 h.Len = h.Cap 459 return 460 } 461 462 // set bucket key 463 func (b *bucket) setKey(m *Map, s string) { 464 var d []byte 465 h := (*reflect.SliceHeader)(unsafe.Pointer(&d)) 466 a := uintptr(unsafe.Pointer(b)) + unsafe.Sizeof(bucket{}) 467 h.Data = a + 1 468 h.Cap = int(m.head.keySize - 1) 469 h.Len = h.Cap 470 l := copy(d, s) 471 *(*uint8)(unsafe.Pointer(a)) = uint8(l) 472 } 473 474 // string hash func 475 func hashFunc(s string) (int32, error) { 476 var b []byte 477 *(*string)(unsafe.Pointer(&b)) = s 478 (*reflect.SliceHeader)(unsafe.Pointer(&b)).Cap = len(s) 479 hs := crc32.NewIEEE() 480 _, err := hs.Write(b) 481 if err != nil { 482 return 0, err 483 } 484 return int32(hs.Sum32()), nil 485 }