github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/pkg/storage/chunk/client/local/boltdb_index_client.go (about) 1 package local 2 3 import ( 4 "bytes" 5 "context" 6 "flag" 7 "fmt" 8 "os" 9 "path" 10 "path/filepath" 11 "sync" 12 "time" 13 14 "github.com/go-kit/log/level" 15 "github.com/pkg/errors" 16 "go.etcd.io/bbolt" 17 18 "github.com/grafana/loki/pkg/storage/chunk/client/util" 19 "github.com/grafana/loki/pkg/storage/stores/series/index" 20 util_log "github.com/grafana/loki/pkg/util/log" 21 ) 22 23 var ( 24 IndexBucketName = []byte("index") 25 ErrUnexistentBoltDB = errors.New("boltdb file does not exist") 26 ErrEmptyIndexBucketName = errors.New("empty index bucket name") 27 ) 28 29 const ( 30 separator = "\000" 31 dbReloadPeriod = 10 * time.Minute 32 33 DBOperationRead = iota 34 DBOperationWrite 35 36 openBoltDBFileTimeout = 5 * time.Second 37 ) 38 39 // BoltDBConfig for a BoltDB index client. 40 type BoltDBConfig struct { 41 Directory string `yaml:"directory"` 42 } 43 44 // RegisterFlags registers flags. 45 func (cfg *BoltDBConfig) RegisterFlags(f *flag.FlagSet) { 46 f.StringVar(&cfg.Directory, "boltdb.dir", "", "Location of BoltDB index files.") 47 } 48 49 type BoltIndexClient struct { 50 cfg BoltDBConfig 51 52 dbsMtx sync.RWMutex 53 dbs map[string]*bbolt.DB 54 done chan struct{} 55 wait sync.WaitGroup 56 } 57 58 // NewBoltDBIndexClient creates a new IndexClient that used BoltDB. 59 func NewBoltDBIndexClient(cfg BoltDBConfig) (*BoltIndexClient, error) { 60 if err := util.EnsureDirectory(cfg.Directory); err != nil { 61 return nil, err 62 } 63 64 indexClient := &BoltIndexClient{ 65 cfg: cfg, 66 dbs: map[string]*bbolt.DB{}, 67 done: make(chan struct{}), 68 } 69 70 indexClient.wait.Add(1) 71 go indexClient.loop() 72 return indexClient, nil 73 } 74 75 func (b *BoltIndexClient) loop() { 76 defer b.wait.Done() 77 78 ticker := time.NewTicker(dbReloadPeriod) 79 defer ticker.Stop() 80 81 for { 82 select { 83 case <-ticker.C: 84 b.reload() 85 case <-b.done: 86 return 87 } 88 } 89 } 90 91 func (b *BoltIndexClient) reload() { 92 b.dbsMtx.RLock() 93 94 removedDBs := []string{} 95 for name := range b.dbs { 96 if _, err := os.Stat(path.Join(b.cfg.Directory, name)); err != nil && os.IsNotExist(err) { 97 removedDBs = append(removedDBs, name) 98 level.Debug(util_log.Logger).Log("msg", "boltdb file got removed", "filename", name) 99 continue 100 } 101 } 102 b.dbsMtx.RUnlock() 103 104 if len(removedDBs) != 0 { 105 b.dbsMtx.Lock() 106 defer b.dbsMtx.Unlock() 107 108 for _, name := range removedDBs { 109 if err := b.dbs[name].Close(); err != nil { 110 level.Error(util_log.Logger).Log("msg", "failed to close removed boltdb", "filename", name, "err", err) 111 continue 112 } 113 delete(b.dbs, name) 114 } 115 } 116 } 117 118 func (b *BoltIndexClient) Stop() { 119 close(b.done) 120 121 b.dbsMtx.Lock() 122 defer b.dbsMtx.Unlock() 123 for _, db := range b.dbs { 124 db.Close() 125 } 126 127 b.wait.Wait() 128 } 129 130 func (b *BoltIndexClient) NewWriteBatch() index.WriteBatch { 131 return NewWriteBatch() 132 } 133 134 func NewWriteBatch() index.WriteBatch { 135 return &BoltWriteBatch{ 136 Writes: map[string]TableWrites{}, 137 } 138 } 139 140 // GetDB should always return a db for write operation unless an error occurs while doing so. 141 // While for read operation it should throw ErrUnexistentBoltDB error if file does not exist for reading 142 func (b *BoltIndexClient) GetDB(name string, operation int) (*bbolt.DB, error) { 143 b.dbsMtx.RLock() 144 db, ok := b.dbs[name] 145 b.dbsMtx.RUnlock() 146 if ok { 147 return db, nil 148 } 149 150 // we do not want to create a new db for reading if it does not exist 151 if operation == DBOperationRead { 152 if _, err := os.Stat(path.Join(b.cfg.Directory, name)); err != nil { 153 if os.IsNotExist(err) { 154 return nil, ErrUnexistentBoltDB 155 } 156 return nil, err 157 } 158 } 159 160 b.dbsMtx.Lock() 161 defer b.dbsMtx.Unlock() 162 db, ok = b.dbs[name] 163 if ok { 164 return db, nil 165 } 166 167 // Open the database. 168 // Set Timeout to avoid obtaining file lock wait indefinitely. 169 db, err := bbolt.Open(path.Join(b.cfg.Directory, name), 0o666, &bbolt.Options{Timeout: openBoltDBFileTimeout}) 170 if err != nil { 171 return nil, fmt.Errorf("failed to open boltdb index file: %w", err) 172 } 173 174 b.dbs[name] = db 175 return db, nil 176 } 177 178 func WriteToDB(_ context.Context, db *bbolt.DB, bucketName []byte, writes TableWrites) error { 179 return db.Update(func(tx *bbolt.Tx) error { 180 var b *bbolt.Bucket 181 if len(bucketName) == 0 { 182 return ErrEmptyIndexBucketName 183 } 184 185 // a bucket should already exist for deletes, for other writes we create one otherwise. 186 if len(writes.deletes) != 0 { 187 b = tx.Bucket(bucketName) 188 if b == nil { 189 return fmt.Errorf("bucket %s not found in table %s", bucketName, filepath.Base(db.Path())) 190 } 191 } else { 192 var err error 193 b, err = tx.CreateBucketIfNotExists(bucketName) 194 if err != nil { 195 return err 196 } 197 } 198 199 for key, value := range writes.puts { 200 if err := b.Put([]byte(key), value); err != nil { 201 return err 202 } 203 } 204 205 for key := range writes.deletes { 206 if err := b.Delete([]byte(key)); err != nil { 207 return err 208 } 209 } 210 211 return nil 212 }) 213 } 214 215 func (b *BoltIndexClient) BatchWrite(ctx context.Context, batch index.WriteBatch) error { 216 for table, writes := range batch.(*BoltWriteBatch).Writes { 217 db, err := b.GetDB(table, DBOperationWrite) 218 if err != nil { 219 return err 220 } 221 222 err = WriteToDB(ctx, db, IndexBucketName, writes) 223 if err != nil { 224 return err 225 } 226 } 227 228 return nil 229 } 230 231 func (b *BoltIndexClient) QueryPages(ctx context.Context, queries []index.Query, callback index.QueryPagesCallback) error { 232 return util.DoParallelQueries(ctx, b.query, queries, callback) 233 } 234 235 func (b *BoltIndexClient) query(ctx context.Context, query index.Query, callback index.QueryPagesCallback) error { 236 db, err := b.GetDB(query.TableName, DBOperationRead) 237 if err != nil { 238 if err == ErrUnexistentBoltDB { 239 return nil 240 } 241 242 return err 243 } 244 245 return QueryDB(ctx, db, IndexBucketName, query, callback) 246 } 247 248 func QueryDB(ctx context.Context, db *bbolt.DB, bucketName []byte, query index.Query, 249 callback index.QueryPagesCallback, 250 ) error { 251 return db.View(func(tx *bbolt.Tx) error { 252 if len(bucketName) == 0 { 253 return ErrEmptyIndexBucketName 254 } 255 bucket := tx.Bucket(bucketName) 256 if bucket == nil { 257 return nil 258 } 259 260 return QueryWithCursor(ctx, bucket.Cursor(), query, callback) 261 }) 262 } 263 264 func QueryWithCursor(_ context.Context, c *bbolt.Cursor, query index.Query, callback index.QueryPagesCallback) error { 265 batch := batchPool.Get().(*cursorBatch) 266 defer batchPool.Put(batch) 267 268 batch.reset(c, &query) 269 callback(query, batch) 270 return nil 271 } 272 273 var batchPool = sync.Pool{ 274 New: func() interface{} { 275 return &cursorBatch{ 276 start: bytes.NewBuffer(make([]byte, 0, 1024)), 277 rowPrefix: bytes.NewBuffer(make([]byte, 0, 1024)), 278 } 279 }, 280 } 281 282 type cursorBatch struct { 283 cursor *bbolt.Cursor 284 query *index.Query 285 start *bytes.Buffer 286 rowPrefix *bytes.Buffer 287 seeked bool 288 289 currRangeValue []byte 290 currValue []byte 291 } 292 293 func (c *cursorBatch) Iterator() index.ReadBatchIterator { 294 return c 295 } 296 297 func (c *cursorBatch) nextItem() ([]byte, []byte) { 298 if !c.seeked { 299 if len(c.query.RangeValuePrefix) > 0 { 300 c.start.WriteString(c.query.HashValue) 301 c.start.WriteString(separator) 302 c.start.Write(c.query.RangeValuePrefix) 303 } else if len(c.query.RangeValueStart) > 0 { 304 c.start.WriteString(c.query.HashValue) 305 c.start.WriteString(separator) 306 c.start.Write(c.query.RangeValueStart) 307 } else { 308 c.start.WriteString(c.query.HashValue) 309 c.start.WriteString(separator) 310 } 311 c.rowPrefix.WriteString(c.query.HashValue) 312 c.rowPrefix.WriteString(separator) 313 c.seeked = true 314 return c.cursor.Seek(c.start.Bytes()) 315 } 316 return c.cursor.Next() 317 } 318 319 func (c *cursorBatch) Next() bool { 320 for k, v := c.nextItem(); k != nil; k, v = c.nextItem() { 321 if !bytes.HasPrefix(k, c.rowPrefix.Bytes()) { 322 break 323 } 324 325 if len(c.query.RangeValuePrefix) > 0 && !bytes.HasPrefix(k, c.start.Bytes()) { 326 break 327 } 328 if len(c.query.ValueEqual) > 0 && !bytes.Equal(v, c.query.ValueEqual) { 329 continue 330 } 331 332 // make a copy since k, v are only valid for the life of the transaction. 333 // See: https://godoc.org/github.com/boltdb/bolt#Cursor.Seek 334 rangeValue := make([]byte, len(k)-c.rowPrefix.Len()) 335 copy(rangeValue, k[c.rowPrefix.Len():]) 336 337 value := make([]byte, len(v)) 338 copy(value, v) 339 340 c.currRangeValue = rangeValue 341 c.currValue = value 342 return true 343 } 344 return false 345 } 346 347 func (c *cursorBatch) RangeValue() []byte { 348 return c.currRangeValue 349 } 350 351 func (c *cursorBatch) Value() []byte { 352 return c.currValue 353 } 354 355 func (c *cursorBatch) reset(cur *bbolt.Cursor, q *index.Query) { 356 c.currRangeValue = nil 357 c.currValue = nil 358 c.seeked = false 359 c.cursor = cur 360 c.query = q 361 c.rowPrefix.Reset() 362 c.start.Reset() 363 } 364 365 type TableWrites struct { 366 puts map[string][]byte 367 deletes map[string]struct{} 368 } 369 370 type BoltWriteBatch struct { 371 Writes map[string]TableWrites 372 } 373 374 func (b *BoltWriteBatch) getOrCreateTableWrites(tableName string) TableWrites { 375 writes, ok := b.Writes[tableName] 376 if !ok { 377 writes = TableWrites{ 378 puts: map[string][]byte{}, 379 deletes: map[string]struct{}{}, 380 } 381 b.Writes[tableName] = writes 382 } 383 384 return writes 385 } 386 387 func (b *BoltWriteBatch) Delete(tableName, hashValue string, rangeValue []byte) { 388 writes := b.getOrCreateTableWrites(tableName) 389 390 key := hashValue + separator + string(rangeValue) 391 writes.deletes[key] = struct{}{} 392 } 393 394 func (b *BoltWriteBatch) Add(tableName, hashValue string, rangeValue []byte, value []byte) { 395 writes := b.getOrCreateTableWrites(tableName) 396 397 key := hashValue + separator + string(rangeValue) 398 writes.puts[key] = value 399 } 400 401 // Open the database. 402 // Set Timeout to avoid obtaining file lock wait indefinitely. 403 func OpenBoltdbFile(path string) (*bbolt.DB, error) { 404 return bbolt.Open(path, 0o666, &bbolt.Options{Timeout: 5 * time.Second}) 405 }