github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/resource_use.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package db
    13  
    14  import (
    15  	"fmt"
    16  	"time"
    17  
    18  	enterrors "github.com/weaviate/weaviate/entities/errors"
    19  
    20  	"github.com/weaviate/weaviate/entities/interval"
    21  	"github.com/weaviate/weaviate/entities/storagestate"
    22  	"github.com/weaviate/weaviate/usecases/memwatch"
    23  )
    24  
    25  type diskUse struct {
    26  	total uint64
    27  	free  uint64
    28  	avail uint64
    29  }
    30  
    31  func (d diskUse) percentUsed() float64 {
    32  	used := d.total - d.free
    33  	return (float64(used) / float64(d.total)) * 100
    34  }
    35  
    36  func (d diskUse) String() string {
    37  	GB := 1024 * 1024 * 1024
    38  
    39  	return fmt.Sprintf("total: %.2fGB, free: %.2fGB, used: %.2fGB (avail: %.2fGB)",
    40  		float64(d.total)/float64(GB),
    41  		float64(d.free)/float64(GB),
    42  		float64(d.total-d.free)/float64(GB),
    43  		float64(d.avail)/float64(GB))
    44  }
    45  
    46  func (d *DB) scanResourceUsage() {
    47  	f := func() {
    48  		t := time.NewTicker(time.Millisecond * 500)
    49  		defer t.Stop()
    50  		for {
    51  			select {
    52  			case <-d.shutdown:
    53  				return
    54  			case <-t.C:
    55  				if !d.resourceScanState.isReadOnly {
    56  					du := d.getDiskUse(d.config.RootPath)
    57  					d.resourceUseWarn(d.memMonitor, du)
    58  					d.resourceUseReadonly(d.memMonitor, du)
    59  				}
    60  			}
    61  		}
    62  	}
    63  	enterrors.GoWrapper(f, d.logger)
    64  }
    65  
    66  type resourceScanState struct {
    67  	diskWarning *interval.BackoffTimer
    68  	memWarning  *interval.BackoffTimer
    69  	isReadOnly  bool
    70  }
    71  
    72  func newResourceScanState() *resourceScanState {
    73  	return &resourceScanState{
    74  		diskWarning: interval.NewBackoffTimer(),
    75  		memWarning:  interval.NewBackoffTimer(),
    76  	}
    77  }
    78  
    79  // logs a warning if user-set threshold is surpassed
    80  func (db *DB) resourceUseWarn(mon *memwatch.Monitor, du diskUse) {
    81  	mon.Refresh()
    82  	db.diskUseWarn(du)
    83  	db.memUseWarn(mon)
    84  }
    85  
    86  func (db *DB) diskUseWarn(du diskUse) {
    87  	diskWarnPercent := db.config.ResourceUsage.DiskUse.WarningPercentage
    88  	if diskWarnPercent > 0 {
    89  		if pu := du.percentUsed(); pu > float64(diskWarnPercent) {
    90  			if db.resourceScanState.diskWarning.IntervalElapsed() {
    91  				db.logger.WithField("action", "read_disk_use").
    92  					WithField("path", db.config.RootPath).
    93  					Warnf("disk usage currently at %.2f%%, threshold set to %.2f%%",
    94  						pu, float64(diskWarnPercent))
    95  
    96  				db.logger.WithField("action", "disk_use_stats").
    97  					WithField("path", db.config.RootPath).
    98  					Debugf("%s", du.String())
    99  				db.resourceScanState.diskWarning.IncreaseInterval()
   100  			}
   101  		}
   102  	}
   103  }
   104  
   105  func (db *DB) memUseWarn(mon *memwatch.Monitor) {
   106  	memWarnPercent := db.config.ResourceUsage.MemUse.WarningPercentage
   107  	if memWarnPercent > 0 {
   108  		if pu := mon.Ratio() * 100; pu > float64(memWarnPercent) {
   109  			if db.resourceScanState.memWarning.IntervalElapsed() {
   110  				db.logger.WithField("action", "read_memory_use").
   111  					WithField("path", db.config.RootPath).
   112  					Warnf("memory usage currently at %.2f%%, threshold set to %.2f%%",
   113  						pu, float64(memWarnPercent))
   114  				db.resourceScanState.memWarning.IncreaseInterval()
   115  			}
   116  		}
   117  	}
   118  }
   119  
   120  // sets the shard to readonly if user-set threshold is surpassed
   121  func (db *DB) resourceUseReadonly(mon *memwatch.Monitor, du diskUse) {
   122  	db.diskUseReadonly(du)
   123  	db.memUseReadonly(mon)
   124  }
   125  
   126  func (db *DB) diskUseReadonly(du diskUse) {
   127  	diskROPercent := db.config.ResourceUsage.DiskUse.ReadOnlyPercentage
   128  	if diskROPercent > 0 {
   129  		if pu := du.percentUsed(); pu > float64(diskROPercent) {
   130  			db.setShardsReadOnly()
   131  			db.logger.WithField("action", "set_shard_read_only").
   132  				WithField("path", db.config.RootPath).
   133  				Warnf("Set READONLY, disk usage currently at %.2f%%, threshold set to %.2f%%",
   134  					pu, float64(diskROPercent))
   135  		}
   136  	}
   137  }
   138  
   139  func (db *DB) memUseReadonly(mon *memwatch.Monitor) {
   140  	memROPercent := db.config.ResourceUsage.MemUse.ReadOnlyPercentage
   141  	if memROPercent > 0 {
   142  		if pu := mon.Ratio() * 100; pu > float64(memROPercent) {
   143  			db.setShardsReadOnly()
   144  			db.logger.WithField("action", "set_shard_read_only").
   145  				WithField("path", db.config.RootPath).
   146  				Warnf("Set READONLY, memory usage currently at %.2f%%, threshold set to %.2f%%",
   147  					pu, float64(memROPercent))
   148  		}
   149  	}
   150  }
   151  
   152  func (db *DB) setShardsReadOnly() {
   153  	db.indexLock.Lock()
   154  	for _, index := range db.indices {
   155  		index.ForEachShard(func(name string, shard ShardLike) error {
   156  			err := shard.UpdateStatus(storagestate.StatusReadOnly.String())
   157  			if err != nil {
   158  				db.logger.WithField("action", "set_shard_read_only").
   159  					WithField("path", db.config.RootPath).
   160  					WithError(err).
   161  					Fatal("failed to set to READONLY")
   162  			}
   163  			return nil
   164  		})
   165  	}
   166  	db.indexLock.Unlock()
   167  	db.resourceScanState.isReadOnly = true
   168  }