github.com/flower-corp/rosedb@v1.1.2-0.20230117132829-21dc4f7b319a/list.go (about) 1 package rosedb 2 3 import ( 4 "bytes" 5 "encoding/binary" 6 "github.com/flower-corp/rosedb/ds/art" 7 "github.com/flower-corp/rosedb/logfile" 8 "github.com/flower-corp/rosedb/logger" 9 "math" 10 ) 11 12 // LPush insert all the specified values at the head of the list stored at key. 13 // If key does not exist, it is created as empty list before performing the push operations. 14 func (db *RoseDB) LPush(key []byte, values ...[]byte) error { 15 db.listIndex.mu.Lock() 16 defer db.listIndex.mu.Unlock() 17 18 if db.listIndex.trees[string(key)] == nil { 19 db.listIndex.trees[string(key)] = art.NewART() 20 } 21 for _, val := range values { 22 if err := db.pushInternal(key, val, true); err != nil { 23 return err 24 } 25 } 26 return nil 27 } 28 29 // LPushX insert specified values at the head of the list stored at key, 30 // only if key already exists and holds a list. 31 // In contrary to LPUSH, no operation will be performed when key does not yet exist. 32 func (db *RoseDB) LPushX(key []byte, values ...[]byte) error { 33 db.listIndex.mu.Lock() 34 defer db.listIndex.mu.Unlock() 35 36 if db.listIndex.trees[string(key)] == nil { 37 return ErrKeyNotFound 38 } 39 40 for _, val := range values { 41 if err := db.pushInternal(key, val, true); err != nil { 42 return err 43 } 44 } 45 return nil 46 } 47 48 // RPush insert all the specified values at the tail of the list stored at key. 49 // If key does not exist, it is created as empty list before performing the push operation. 50 func (db *RoseDB) RPush(key []byte, values ...[]byte) error { 51 db.listIndex.mu.Lock() 52 defer db.listIndex.mu.Unlock() 53 54 if db.listIndex.trees[string(key)] == nil { 55 db.listIndex.trees[string(key)] = art.NewART() 56 } 57 for _, val := range values { 58 if err := db.pushInternal(key, val, false); err != nil { 59 return err 60 } 61 } 62 return nil 63 } 64 65 // RPushX insert specified values at the tail of the list stored at key, 66 // only if key already exists and holds a list. 67 // In contrary to RPUSH, no operation will be performed when key does not yet exist. 68 func (db *RoseDB) RPushX(key []byte, values ...[]byte) error { 69 db.listIndex.mu.Lock() 70 defer db.listIndex.mu.Unlock() 71 72 if db.listIndex.trees[string(key)] == nil { 73 return ErrKeyNotFound 74 } 75 for _, val := range values { 76 if err := db.pushInternal(key, val, false); err != nil { 77 return err 78 } 79 } 80 return nil 81 } 82 83 // LPop removes and returns the first elements of the list stored at key. 84 func (db *RoseDB) LPop(key []byte) ([]byte, error) { 85 db.listIndex.mu.Lock() 86 defer db.listIndex.mu.Unlock() 87 return db.popInternal(key, true) 88 } 89 90 // RPop Removes and returns the last elements of the list stored at key. 91 func (db *RoseDB) RPop(key []byte) ([]byte, error) { 92 db.listIndex.mu.Lock() 93 defer db.listIndex.mu.Unlock() 94 return db.popInternal(key, false) 95 } 96 97 // LMove atomically returns and removes the first/last element of the list stored at source, 98 // and pushes the element at the first/last element of the list stored at destination. 99 func (db *RoseDB) LMove(srcKey, dstKey []byte, srcIsLeft, dstIsLeft bool) ([]byte, error) { 100 db.listIndex.mu.Lock() 101 defer db.listIndex.mu.Unlock() 102 103 popValue, err := db.popInternal(srcKey, srcIsLeft) 104 if err != nil { 105 return nil, err 106 } 107 if popValue == nil { 108 return nil, nil 109 } 110 111 if db.listIndex.trees[string(dstKey)] == nil { 112 db.listIndex.trees[string(dstKey)] = art.NewART() 113 } 114 if err = db.pushInternal(dstKey, popValue, dstIsLeft); err != nil { 115 return nil, err 116 } 117 118 return popValue, nil 119 } 120 121 // LLen returns the length of the list stored at key. 122 // If key does not exist, it is interpreted as an empty list and 0 is returned. 123 func (db *RoseDB) LLen(key []byte) int { 124 db.listIndex.mu.RLock() 125 defer db.listIndex.mu.RUnlock() 126 127 if db.listIndex.trees[string(key)] == nil { 128 return 0 129 } 130 idxTree := db.listIndex.trees[string(key)] 131 headSeq, tailSeq, err := db.listMeta(idxTree, key) 132 if err != nil { 133 return 0 134 } 135 return int(tailSeq - headSeq - 1) 136 } 137 138 // LIndex returns the element at index in the list stored at key. 139 // If index is out of range, it returns nil. 140 func (db *RoseDB) LIndex(key []byte, index int) ([]byte, error) { 141 db.listIndex.mu.RLock() 142 defer db.listIndex.mu.RUnlock() 143 144 if db.listIndex.trees[string(key)] == nil { 145 return nil, nil 146 } 147 idxTree := db.listIndex.trees[string(key)] 148 headSeq, tailSeq, err := db.listMeta(idxTree, key) 149 if err != nil { 150 return nil, err 151 } 152 153 seq, err := db.listSequence(headSeq, tailSeq, index) 154 if err != nil { 155 return nil, err 156 } 157 158 if seq >= tailSeq || seq <= headSeq { 159 return nil, ErrWrongIndex 160 } 161 162 encKey := db.encodeListKey(key, seq) 163 val, err := db.getVal(idxTree, encKey, List) 164 if err != nil { 165 return nil, err 166 } 167 return val, nil 168 } 169 170 // LSet Sets the list element at index to element. 171 func (db *RoseDB) LSet(key []byte, index int, value []byte) error { 172 db.listIndex.mu.Lock() 173 defer db.listIndex.mu.Unlock() 174 175 if db.listIndex.trees[string(key)] == nil { 176 return ErrKeyNotFound 177 } 178 idxTree := db.listIndex.trees[string(key)] 179 headSeq, tailSeq, err := db.listMeta(idxTree, key) 180 if err != nil { 181 return err 182 } 183 184 seq, err := db.listSequence(headSeq, tailSeq, index) 185 if err != nil { 186 return err 187 } 188 189 if seq >= tailSeq || seq <= headSeq { 190 return ErrWrongIndex 191 } 192 193 encKey := db.encodeListKey(key, seq) 194 ent := &logfile.LogEntry{Key: encKey, Value: value} 195 valuePos, err := db.writeLogEntry(ent, List) 196 if err != nil { 197 return err 198 } 199 if err = db.updateIndexTree(idxTree, ent, valuePos, true, List); err != nil { 200 return err 201 } 202 return nil 203 } 204 205 // LRange returns the specified elements of the list stored at key. 206 // The offsets start and stop are zero-based indexes, with 0 being the first element 207 // of the list (the head of the list), 1 being the next element and so on. 208 // These offsets can also be negative numbers indicating offsets starting at the end of the list. 209 // For example, -1 is the last element of the list, -2 the penultimate, and so on. 210 // If start is larger than the end of the list, an empty list is returned. 211 // If stop is larger than the actual end of the list, Redis will treat it like the last element of the list. 212 func (db *RoseDB) LRange(key []byte, start, end int) (values [][]byte, err error) { 213 db.listIndex.mu.RLock() 214 defer db.listIndex.mu.RUnlock() 215 216 if db.listIndex.trees[string(key)] == nil { 217 return nil, ErrKeyNotFound 218 } 219 220 idxTree := db.listIndex.trees[string(key)] 221 // get List DataType meta info 222 headSeq, tailSeq, err := db.listMeta(idxTree, key) 223 if err != nil { 224 return nil, err 225 } 226 227 var startSeq, endSeq uint32 228 229 // logical address to physical address 230 startSeq, err = db.listSequence(headSeq, tailSeq, start) 231 if err != nil { 232 return nil, err 233 } 234 endSeq, err = db.listSequence(headSeq, tailSeq, end) 235 if err != nil { 236 return nil, err 237 } 238 // normalize startSeq 239 if startSeq <= headSeq { 240 startSeq = headSeq + 1 241 } 242 243 // normalize endSeq 244 if endSeq >= tailSeq { 245 endSeq = tailSeq - 1 246 } 247 248 if startSeq >= tailSeq || endSeq <= headSeq || startSeq > endSeq { 249 return nil, ErrWrongIndex 250 } 251 252 // the endSeq value is included 253 for seq := startSeq; seq < endSeq+1; seq++ { 254 encKey := db.encodeListKey(key, seq) 255 val, err := db.getVal(idxTree, encKey, List) 256 257 if err != nil { 258 return nil, err 259 } 260 values = append(values, val) 261 } 262 return values, nil 263 } 264 265 func (db *RoseDB) encodeListKey(key []byte, seq uint32) []byte { 266 buf := make([]byte, len(key)+4) 267 binary.LittleEndian.PutUint32(buf[:4], seq) 268 copy(buf[4:], key[:]) 269 return buf 270 } 271 272 func (db *RoseDB) decodeListKey(buf []byte) ([]byte, uint32) { 273 seq := binary.LittleEndian.Uint32(buf[:4]) 274 key := make([]byte, len(buf[4:])) 275 copy(key[:], buf[4:]) 276 return key, seq 277 } 278 279 func (db *RoseDB) listMeta(idxTree *art.AdaptiveRadixTree, key []byte) (uint32, uint32, error) { 280 val, err := db.getVal(idxTree, key, List) 281 if err != nil && err != ErrKeyNotFound { 282 return 0, 0, err 283 } 284 285 var headSeq uint32 = initialListSeq 286 var tailSeq uint32 = initialListSeq + 1 287 if len(val) != 0 { 288 headSeq = binary.LittleEndian.Uint32(val[:4]) 289 tailSeq = binary.LittleEndian.Uint32(val[4:8]) 290 } 291 return headSeq, tailSeq, nil 292 } 293 294 func (db *RoseDB) saveListMeta(idxTree *art.AdaptiveRadixTree, key []byte, headSeq, tailSeq uint32) error { 295 buf := make([]byte, 8) 296 binary.LittleEndian.PutUint32(buf[:4], headSeq) 297 binary.LittleEndian.PutUint32(buf[4:8], tailSeq) 298 ent := &logfile.LogEntry{Key: key, Value: buf, Type: logfile.TypeListMeta} 299 pos, err := db.writeLogEntry(ent, List) 300 if err != nil { 301 return err 302 } 303 err = db.updateIndexTree(idxTree, ent, pos, true, List) 304 return err 305 } 306 307 func (db *RoseDB) pushInternal(key []byte, val []byte, isLeft bool) error { 308 idxTree := db.listIndex.trees[string(key)] 309 headSeq, tailSeq, err := db.listMeta(idxTree, key) 310 if err != nil { 311 return err 312 } 313 var seq = headSeq 314 if !isLeft { 315 seq = tailSeq 316 } 317 encKey := db.encodeListKey(key, seq) 318 ent := &logfile.LogEntry{Key: encKey, Value: val} 319 valuePos, err := db.writeLogEntry(ent, List) 320 if err != nil { 321 return err 322 } 323 324 if err = db.updateIndexTree(idxTree, ent, valuePos, true, List); err != nil { 325 return err 326 } 327 328 if isLeft { 329 headSeq-- 330 } else { 331 tailSeq++ 332 } 333 err = db.saveListMeta(idxTree, key, headSeq, tailSeq) 334 return err 335 } 336 337 func (db *RoseDB) popInternal(key []byte, isLeft bool) ([]byte, error) { 338 if db.listIndex.trees[string(key)] == nil { 339 return nil, nil 340 } 341 idxTree := db.listIndex.trees[string(key)] 342 headSeq, tailSeq, err := db.listMeta(idxTree, key) 343 if err != nil { 344 return nil, err 345 } 346 if tailSeq-headSeq-1 <= 0 { 347 return nil, nil 348 } 349 350 var seq = headSeq + 1 351 if !isLeft { 352 seq = tailSeq - 1 353 } 354 encKey := db.encodeListKey(key, seq) 355 val, err := db.getVal(idxTree, encKey, List) 356 if err != nil { 357 return nil, err 358 } 359 360 ent := &logfile.LogEntry{Key: encKey, Type: logfile.TypeDelete} 361 pos, err := db.writeLogEntry(ent, List) 362 if err != nil { 363 return nil, err 364 } 365 oldVal, updated := idxTree.Delete(encKey) 366 if isLeft { 367 headSeq++ 368 } else { 369 tailSeq-- 370 } 371 if err = db.saveListMeta(idxTree, key, headSeq, tailSeq); err != nil { 372 return nil, err 373 } 374 // send discard 375 db.sendDiscard(oldVal, updated, List) 376 _, entrySize := logfile.EncodeEntry(ent) 377 node := &indexNode{fid: pos.fid, entrySize: entrySize} 378 select { 379 case db.discards[List].valChan <- node: 380 default: 381 logger.Warn("send to discard chan fail") 382 } 383 if tailSeq-headSeq-1 == 0 { 384 // reset meta 385 if headSeq != initialListSeq || tailSeq != initialListSeq+1 { 386 headSeq = initialListSeq 387 tailSeq = initialListSeq + 1 388 _ = db.saveListMeta(idxTree, key, headSeq, tailSeq) 389 } 390 delete(db.listIndex.trees, string(key)) 391 } 392 return val, nil 393 } 394 395 // listSequence just convert logical index to physical seq. 396 // whether physical seq is legal or not, just convert it 397 func (db *RoseDB) listSequence(headSeq, tailSeq uint32, index int) (uint32, error) { 398 var seq uint32 399 400 if index >= 0 { 401 seq = headSeq + uint32(index) + 1 402 } else { 403 seq = tailSeq - uint32(-index) 404 } 405 return seq, nil 406 } 407 408 // LRem removes the first count occurrences of elements equal to element from the list stored at key. 409 // The count argument influences the operation in the following ways: 410 // count > 0: Remove elements equal to element moving from head to tail. 411 // count < 0: Remove elements equal to element moving from tail to head. 412 // count = 0: Remove all elements equal to element. 413 // Note that this method will rewrite the values, so it maybe very slow. 414 func (db *RoseDB) LRem(key []byte, count int, value []byte) (int, error) { 415 db.listIndex.mu.Lock() 416 defer db.listIndex.mu.Unlock() 417 418 if count == 0 { 419 count = math.MaxUint32 420 } 421 var discardCount int 422 idxTree := db.listIndex.trees[string(key)] 423 if idxTree == nil { 424 return discardCount, nil 425 } 426 // get List DataType meta info 427 headSeq, tailSeq, err := db.listMeta(idxTree, key) 428 if err != nil { 429 return discardCount, err 430 } 431 432 reserveSeq, discardSeq, reserveValueSeq := make([]uint32, 0), make([]uint32, 0), make([][]byte, 0) 433 classifyData := func(key []byte, seq uint32) error { 434 encKey := db.encodeListKey(key, seq) 435 val, err := db.getVal(idxTree, encKey, List) 436 if err != nil { 437 return err 438 } 439 if bytes.Equal(value, val) { 440 discardSeq = append(discardSeq, seq) 441 discardCount++ 442 } else { 443 reserveSeq = append(reserveSeq, seq) 444 temp := make([]byte, len(val)) 445 copy(temp, val) 446 reserveValueSeq = append(reserveValueSeq, temp) 447 } 448 return nil 449 } 450 451 addReserveData := func(key []byte, value []byte, isLeft bool) error { 452 if db.listIndex.trees[string(key)] == nil { 453 db.listIndex.trees[string(key)] = art.NewART() 454 } 455 if err := db.pushInternal(key, value, isLeft); err != nil { 456 return err 457 } 458 return nil 459 } 460 461 if count > 0 { 462 // record discard data and reserve data 463 for seq := headSeq + 1; seq < tailSeq; seq++ { 464 if err := classifyData(key, seq); err != nil { 465 return discardCount, err 466 } 467 if discardCount == count { 468 break 469 } 470 } 471 472 discardSeqLen := len(discardSeq) 473 if discardSeqLen > 0 { 474 // delete discard data 475 for seq := headSeq + 1; seq <= discardSeq[discardSeqLen-1]; seq++ { 476 if _, err := db.popInternal(key, true); err != nil { 477 return discardCount, err 478 } 479 } 480 // add reserve data 481 for i := len(reserveSeq) - 1; i >= 0; i-- { 482 if reserveSeq[i] < discardSeq[discardSeqLen-1] { 483 if err := addReserveData(key, reserveValueSeq[i], true); err != nil { 484 return discardCount, err 485 } 486 } 487 } 488 } 489 } else { 490 count = -count 491 // record discard data and reserve data 492 for seq := tailSeq - 1; seq > headSeq; seq-- { 493 if err := classifyData(key, seq); err != nil { 494 return discardCount, err 495 } 496 if discardCount == count { 497 break 498 } 499 } 500 501 discardSeqLen := len(discardSeq) 502 if discardSeqLen > 0 { 503 // delete discard data 504 for seq := tailSeq - 1; seq >= discardSeq[discardSeqLen-1]; seq-- { 505 if _, err := db.popInternal(key, false); err != nil { 506 return discardCount, err 507 } 508 } 509 // add reserve data 510 for i := len(reserveSeq) - 1; i >= 0; i-- { 511 if reserveSeq[i] > discardSeq[discardSeqLen-1] { 512 if err := addReserveData(key, reserveValueSeq[i], false); err != nil { 513 return discardCount, err 514 } 515 } 516 } 517 } 518 } 519 return discardCount, nil 520 }