knative.dev/pkg@v0.0.0-20260602142205-ac97e43f6622/controller/queue_metrics.go (about)

     1  /*
     2  Copyright 2025 The Knative Authors
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package controller
    18  
    19  import (
    20  	"sync"
    21  	"time"
    22  
    23  	"k8s.io/client-go/util/workqueue"
    24  	"k8s.io/utils/clock"
    25  	"knative.dev/pkg/observability/metrics/k8s"
    26  )
    27  
    28  // Copyright The Kubernetes Authors.
    29  // Forked from: https://github.com/kubernetes/client-go/blob/release-1.33/util/workqueue/metrics.go
    30  
    31  var (
    32  	noopProvider = k8s.NewNoopWorkqueueMetricsProvider()
    33  
    34  	// Unfortunately k8s package doesn't expose this variable so for now
    35  	// we'll create our own and potentially upstream some changes in the future.
    36  	globalMetricsProvider        workqueue.MetricsProvider = noopProvider
    37  	setGlobalMetricsProviderOnce sync.Once
    38  )
    39  
    40  func SetMetricsProvider(metricsProvider workqueue.MetricsProvider) {
    41  	setGlobalMetricsProviderOnce.Do(func() {
    42  		globalMetricsProvider = metricsProvider
    43  	})
    44  }
    45  
    46  // queueMetrics expects the caller to lock before setting any metrics.
    47  type queueMetrics struct {
    48  	clock clock.Clock
    49  
    50  	// current depth of a workqueue
    51  	depth workqueue.GaugeMetric
    52  	// total number of adds handled by a workqueue
    53  	adds workqueue.CounterMetric
    54  	// how long an item stays in a workqueue
    55  	latency workqueue.HistogramMetric
    56  	// how long processing an item from a workqueue takes
    57  	workDuration         workqueue.HistogramMetric
    58  	addTimes             map[any]time.Time
    59  	processingStartTimes map[any]time.Time
    60  
    61  	mu sync.Mutex
    62  
    63  	// how long have current threads been working?
    64  	unfinishedWorkSeconds   workqueue.SettableGaugeMetric
    65  	longestRunningProcessor workqueue.SettableGaugeMetric
    66  }
    67  
    68  func (m *queueMetrics) add(item any) {
    69  	if m == nil {
    70  		return
    71  	}
    72  
    73  	m.mu.Lock()
    74  	defer m.mu.Unlock()
    75  
    76  	if _, exists := m.addTimes[item]; !exists {
    77  		m.adds.Inc()
    78  		m.depth.Inc()
    79  		m.addTimes[item] = m.clock.Now()
    80  	}
    81  }
    82  
    83  func (m *queueMetrics) get(item any) {
    84  	if m == nil {
    85  		return
    86  	}
    87  
    88  	m.mu.Lock()
    89  	defer m.mu.Unlock()
    90  
    91  	if startTime, exists := m.addTimes[item]; exists {
    92  		m.depth.Dec()
    93  		m.latency.Observe(m.sinceInSeconds(startTime))
    94  		delete(m.addTimes, item)
    95  	}
    96  
    97  	if _, exists := m.processingStartTimes[item]; !exists {
    98  		m.processingStartTimes[item] = m.clock.Now()
    99  	}
   100  }
   101  
   102  func (m *queueMetrics) done(item any) {
   103  	if m == nil {
   104  		return
   105  	}
   106  
   107  	m.mu.Lock()
   108  	defer m.mu.Unlock()
   109  
   110  	if startTime, exists := m.processingStartTimes[item]; exists {
   111  		m.workDuration.Observe(m.sinceInSeconds(startTime))
   112  		delete(m.processingStartTimes, item)
   113  	}
   114  }
   115  
   116  func (m *queueMetrics) updateUnfinishedWork() {
   117  	if m == nil {
   118  		return
   119  	}
   120  	// Note that a summary metric would be better for this, but prometheus
   121  	// doesn't seem to have non-hacky ways to reset the summary metrics.
   122  	var total float64
   123  	var oldest float64
   124  
   125  	m.mu.Lock()
   126  	defer m.mu.Unlock()
   127  
   128  	for _, t := range m.processingStartTimes {
   129  		age := m.sinceInSeconds(t)
   130  		total += age
   131  		if age > oldest {
   132  			oldest = age
   133  		}
   134  	}
   135  	m.unfinishedWorkSeconds.Set(total)
   136  	m.longestRunningProcessor.Set(oldest)
   137  }
   138  
   139  func (m *queueMetrics) sinceInSeconds(start time.Time) float64 {
   140  	return m.clock.Since(start).Seconds()
   141  }