github.com/flower-corp/rosedb@v1.1.2-0.20230117132829-21dc4f7b319a/index.go (about) 1 package rosedb 2 3 import ( 4 "github.com/flower-corp/rosedb/ds/art" 5 "github.com/flower-corp/rosedb/logfile" 6 "github.com/flower-corp/rosedb/logger" 7 "github.com/flower-corp/rosedb/util" 8 "io" 9 "sort" 10 "sync" 11 "sync/atomic" 12 "time" 13 ) 14 15 // DataType Define the data structure type. 16 type DataType = int8 17 18 // Five different data types, support String, List, Hash, Set, Sorted Set right now. 19 const ( 20 String DataType = iota 21 List 22 Hash 23 Set 24 ZSet 25 ) 26 27 func (db *RoseDB) buildIndex(dataType DataType, ent *logfile.LogEntry, pos *valuePos) { 28 switch dataType { 29 case String: 30 db.buildStrsIndex(ent, pos) 31 case List: 32 db.buildListIndex(ent, pos) 33 case Hash: 34 db.buildHashIndex(ent, pos) 35 case Set: 36 db.buildSetsIndex(ent, pos) 37 case ZSet: 38 db.buildZSetIndex(ent, pos) 39 } 40 } 41 42 func (db *RoseDB) buildStrsIndex(ent *logfile.LogEntry, pos *valuePos) { 43 ts := time.Now().Unix() 44 if ent.Type == logfile.TypeDelete || (ent.ExpiredAt != 0 && ent.ExpiredAt < ts) { 45 db.strIndex.idxTree.Delete(ent.Key) 46 return 47 } 48 _, size := logfile.EncodeEntry(ent) 49 idxNode := &indexNode{fid: pos.fid, offset: pos.offset, entrySize: size} 50 if db.opts.IndexMode == KeyValueMemMode { 51 idxNode.value = ent.Value 52 } 53 if ent.ExpiredAt != 0 { 54 idxNode.expiredAt = ent.ExpiredAt 55 } 56 db.strIndex.idxTree.Put(ent.Key, idxNode) 57 } 58 59 func (db *RoseDB) buildListIndex(ent *logfile.LogEntry, pos *valuePos) { 60 var listKey = ent.Key 61 if ent.Type != logfile.TypeListMeta { 62 listKey, _ = db.decodeListKey(ent.Key) 63 } 64 if db.listIndex.trees[string(listKey)] == nil { 65 db.listIndex.trees[string(listKey)] = art.NewART() 66 } 67 idxTree := db.listIndex.trees[string(listKey)] 68 69 if ent.Type == logfile.TypeDelete { 70 idxTree.Delete(ent.Key) 71 return 72 } 73 _, size := logfile.EncodeEntry(ent) 74 idxNode := &indexNode{fid: pos.fid, offset: pos.offset, entrySize: size} 75 if db.opts.IndexMode == KeyValueMemMode { 76 idxNode.value = ent.Value 77 } 78 if ent.ExpiredAt != 0 { 79 idxNode.expiredAt = ent.ExpiredAt 80 } 81 idxTree.Put(ent.Key, idxNode) 82 } 83 84 func (db *RoseDB) buildHashIndex(ent *logfile.LogEntry, pos *valuePos) { 85 key, field := db.decodeKey(ent.Key) 86 if db.hashIndex.trees[string(key)] == nil { 87 db.hashIndex.trees[string(key)] = art.NewART() 88 } 89 idxTree := db.hashIndex.trees[string(key)] 90 91 if ent.Type == logfile.TypeDelete { 92 idxTree.Delete(field) 93 return 94 } 95 96 _, size := logfile.EncodeEntry(ent) 97 idxNode := &indexNode{fid: pos.fid, offset: pos.offset, entrySize: size} 98 if db.opts.IndexMode == KeyValueMemMode { 99 idxNode.value = ent.Value 100 } 101 if ent.ExpiredAt != 0 { 102 idxNode.expiredAt = ent.ExpiredAt 103 } 104 idxTree.Put(field, idxNode) 105 } 106 107 func (db *RoseDB) buildSetsIndex(ent *logfile.LogEntry, pos *valuePos) { 108 if db.setIndex.trees[string(ent.Key)] == nil { 109 db.setIndex.trees[string(ent.Key)] = art.NewART() 110 } 111 idxTree := db.setIndex.trees[string(ent.Key)] 112 113 if ent.Type == logfile.TypeDelete { 114 idxTree.Delete(ent.Value) 115 return 116 } 117 118 if err := db.setIndex.murhash.Write(ent.Value); err != nil { 119 logger.Fatalf("fail to write murmur hash: %v", err) 120 } 121 sum := db.setIndex.murhash.EncodeSum128() 122 db.setIndex.murhash.Reset() 123 124 _, size := logfile.EncodeEntry(ent) 125 idxNode := &indexNode{fid: pos.fid, offset: pos.offset, entrySize: size} 126 if db.opts.IndexMode == KeyValueMemMode { 127 idxNode.value = ent.Value 128 } 129 if ent.ExpiredAt != 0 { 130 idxNode.expiredAt = ent.ExpiredAt 131 } 132 idxTree.Put(sum, idxNode) 133 } 134 135 func (db *RoseDB) buildZSetIndex(ent *logfile.LogEntry, pos *valuePos) { 136 if ent.Type == logfile.TypeDelete { 137 db.zsetIndex.indexes.ZRem(string(ent.Key), string(ent.Value)) 138 if db.zsetIndex.trees[string(ent.Key)] != nil { 139 db.zsetIndex.trees[string(ent.Key)].Delete(ent.Value) 140 } 141 return 142 } 143 144 key, scoreBuf := db.decodeKey(ent.Key) 145 score, _ := util.StrToFloat64(string(scoreBuf)) 146 if err := db.zsetIndex.murhash.Write(ent.Value); err != nil { 147 logger.Fatalf("fail to write murmur hash: %v", err) 148 } 149 sum := db.zsetIndex.murhash.EncodeSum128() 150 db.zsetIndex.murhash.Reset() 151 152 idxTree := db.zsetIndex.trees[string(key)] 153 if idxTree == nil { 154 idxTree = art.NewART() 155 db.zsetIndex.trees[string(key)] = idxTree 156 } 157 158 _, size := logfile.EncodeEntry(ent) 159 idxNode := &indexNode{fid: pos.fid, offset: pos.offset, entrySize: size} 160 if db.opts.IndexMode == KeyValueMemMode { 161 idxNode.value = ent.Value 162 } 163 if ent.ExpiredAt != 0 { 164 idxNode.expiredAt = ent.ExpiredAt 165 } 166 db.zsetIndex.indexes.ZAdd(string(key), score, string(sum)) 167 idxTree.Put(sum, idxNode) 168 } 169 170 func (db *RoseDB) loadIndexFromLogFiles() error { 171 iterateAndHandle := func(dataType DataType, wg *sync.WaitGroup) { 172 defer wg.Done() 173 174 fids := db.fidMap[dataType] 175 if len(fids) == 0 { 176 return 177 } 178 sort.Slice(fids, func(i, j int) bool { 179 return fids[i] < fids[j] 180 }) 181 182 for i, fid := range fids { 183 var logFile *logfile.LogFile 184 if i == len(fids)-1 { 185 logFile = db.activeLogFiles[dataType] 186 } else { 187 logFile = db.archivedLogFiles[dataType][fid] 188 } 189 if logFile == nil { 190 logger.Fatalf("log file is nil, failed to open db") 191 } 192 193 var offset int64 194 for { 195 entry, esize, err := logFile.ReadLogEntry(offset) 196 if err != nil { 197 if err == io.EOF || err == logfile.ErrEndOfEntry { 198 break 199 } 200 logger.Fatalf("read log entry from file err, failed to open db") 201 } 202 pos := &valuePos{fid: fid, offset: offset} 203 db.buildIndex(dataType, entry, pos) 204 offset += esize 205 } 206 // set latest log file`s WriteAt. 207 if i == len(fids)-1 { 208 atomic.StoreInt64(&logFile.WriteAt, offset) 209 } 210 } 211 } 212 213 wg := new(sync.WaitGroup) 214 wg.Add(logFileTypeNum) 215 for i := 0; i < logFileTypeNum; i++ { 216 go iterateAndHandle(DataType(i), wg) 217 } 218 wg.Wait() 219 return nil 220 } 221 222 func (db *RoseDB) updateIndexTree(idxTree *art.AdaptiveRadixTree, 223 ent *logfile.LogEntry, pos *valuePos, sendDiscard bool, dType DataType) error { 224 225 var size = pos.entrySize 226 if dType == String || dType == List { 227 _, size = logfile.EncodeEntry(ent) 228 } 229 idxNode := &indexNode{fid: pos.fid, offset: pos.offset, entrySize: size} 230 // in KeyValueMemMode, both key and value will store in memory. 231 if db.opts.IndexMode == KeyValueMemMode { 232 idxNode.value = ent.Value 233 } 234 if ent.ExpiredAt != 0 { 235 idxNode.expiredAt = ent.ExpiredAt 236 } 237 238 oldVal, updated := idxTree.Put(ent.Key, idxNode) 239 if sendDiscard { 240 db.sendDiscard(oldVal, updated, dType) 241 } 242 return nil 243 } 244 245 // get index node info from an adaptive radix tree in memory. 246 func (db *RoseDB) getIndexNode(idxTree *art.AdaptiveRadixTree, key []byte) (*indexNode, error) { 247 rawValue := idxTree.Get(key) 248 if rawValue == nil { 249 return nil, ErrKeyNotFound 250 } 251 idxNode, _ := rawValue.(*indexNode) 252 if idxNode == nil { 253 return nil, ErrKeyNotFound 254 } 255 return idxNode, nil 256 } 257 258 func (db *RoseDB) getVal(idxTree *art.AdaptiveRadixTree, 259 key []byte, dataType DataType) ([]byte, error) { 260 261 // Get index info from an adaptive radix tree in memory. 262 rawValue := idxTree.Get(key) 263 if rawValue == nil { 264 return nil, ErrKeyNotFound 265 } 266 idxNode, _ := rawValue.(*indexNode) 267 if idxNode == nil { 268 return nil, ErrKeyNotFound 269 } 270 271 ts := time.Now().Unix() 272 if idxNode.expiredAt != 0 && idxNode.expiredAt <= ts { 273 return nil, ErrKeyNotFound 274 } 275 // In KeyValueMemMode, the value will be stored in memory. 276 // So get the value from the index info. 277 if db.opts.IndexMode == KeyValueMemMode && len(idxNode.value) != 0 { 278 return idxNode.value, nil 279 } 280 281 // In KeyOnlyMemMode, the value not in memory, so get the value from log file at the offset. 282 logFile := db.getActiveLogFile(dataType) 283 if logFile.Fid != idxNode.fid { 284 logFile = db.getArchivedLogFile(dataType, idxNode.fid) 285 } 286 if logFile == nil { 287 return nil, ErrLogFileNotFound 288 } 289 290 ent, _, err := logFile.ReadLogEntry(idxNode.offset) 291 if err != nil { 292 return nil, err 293 } 294 // key exists, but is invalid(deleted or expired) 295 if ent.Type == logfile.TypeDelete || (ent.ExpiredAt != 0 && ent.ExpiredAt < ts) { 296 return nil, ErrKeyNotFound 297 } 298 return ent.Value, nil 299 }