github.com/weaviate/weaviate@v1.24.6/adapters/repos/db/node_wide_metrics.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package db
    13  
    14  import (
    15  	"time"
    16  
    17  	"github.com/prometheus/client_golang/prometheus"
    18  	"github.com/sirupsen/logrus"
    19  )
    20  
    21  type nodeWideMetricsObserver struct {
    22  	db       *DB
    23  	shutdown chan struct{}
    24  }
    25  
    26  func newNodeWideMetricsObserver(db *DB) *nodeWideMetricsObserver {
    27  	return &nodeWideMetricsObserver{db: db, shutdown: make(chan struct{})}
    28  }
    29  
    30  func (o *nodeWideMetricsObserver) Start() {
    31  	t := time.NewTicker(30 * time.Second)
    32  
    33  	defer t.Stop()
    34  
    35  	for {
    36  		select {
    37  		case <-o.shutdown:
    38  			return
    39  		case <-t.C:
    40  			o.observeIfShardsReady()
    41  		}
    42  	}
    43  }
    44  
    45  func (o *nodeWideMetricsObserver) observeIfShardsReady() {
    46  	o.db.indexLock.RLock()
    47  	defer o.db.indexLock.RUnlock()
    48  
    49  	allShardsReady := true
    50  
    51  	for _, index := range o.db.indices {
    52  		if !index.allShardsReady.Load() {
    53  			allShardsReady = false
    54  			break
    55  		}
    56  	}
    57  
    58  	if !allShardsReady {
    59  		o.db.logger.WithFields(logrus.Fields{
    60  			"action": "skip_observe_node_wide_metrics",
    61  		}).Debugf("skip node-wide metrics, not all shards ready")
    62  		return
    63  	}
    64  
    65  	o.observeUnlocked()
    66  }
    67  
    68  // assumes that the caller already holds a db.indexLock.Rlock()
    69  func (o *nodeWideMetricsObserver) observeUnlocked() {
    70  	start := time.Now()
    71  
    72  	totalObjectCount := 0
    73  	for _, index := range o.db.indices {
    74  		index.ForEachShard(func(name string, shard ShardLike) error {
    75  			totalObjectCount += shard.ObjectCountAsync()
    76  			return nil
    77  		})
    78  	}
    79  
    80  	o.db.promMetrics.ObjectCount.With(prometheus.Labels{
    81  		"class_name": "n/a",
    82  		"shard_name": "n/a",
    83  	}).Set(float64(totalObjectCount))
    84  
    85  	took := time.Since(start)
    86  	o.db.logger.WithFields(logrus.Fields{
    87  		"action":       "observe_node_wide_metrics",
    88  		"took":         took,
    89  		"object_count": totalObjectCount,
    90  	}).Debug("observed node wide metrics")
    91  }
    92  
    93  func (o *nodeWideMetricsObserver) Shutdown() {
    94  	o.shutdown <- struct{}{}
    95  }