sigs.k8s.io/prow@v0.0.0-20240503223140-c5e374dc7eb1/pkg/kube/metrics.go (about)

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package kube
    18  
    19  import (
    20  	"github.com/prometheus/client_golang/prometheus"
    21  	"github.com/sirupsen/logrus"
    22  
    23  	prowapi "sigs.k8s.io/prow/pkg/apis/prowjobs/v1"
    24  )
    25  
    26  var (
    27  	metricLabels = []string{
    28  		// namespace of the job
    29  		"job_namespace",
    30  		// name of the job
    31  		"job_name",
    32  		// type of the prowjob: presubmit, postsubmit, periodic, batch
    33  		"type",
    34  		// state of the prowjob: triggered, pending, success, failure, aborted, error
    35  		"state",
    36  		// the org of the prowjob's repo
    37  		"org",
    38  		// the prowjob's repo
    39  		"repo",
    40  		// the base_ref of the prowjob's repo
    41  		"base_ref",
    42  		// the cluster the job runs on
    43  		"cluster",
    44  		// whether or not the job was a retest
    45  		"retest",
    46  	}
    47  	prowJobs = prometheus.NewGaugeVec(prometheus.GaugeOpts{
    48  		Name: "prowjobs",
    49  		Help: "Number of prowjobs in the system.",
    50  	}, metricLabels)
    51  	prowJobTransitions = prometheus.NewCounterVec(prometheus.CounterOpts{
    52  		Name: "prowjob_state_transitions",
    53  		Help: "Number of prowjobs transitioning states.",
    54  	}, metricLabels)
    55  )
    56  
    57  type jobLabel struct {
    58  	jobNamespace string
    59  	jobName      string
    60  	jobType      string
    61  	state        string
    62  	org          string
    63  	repo         string
    64  	baseRef      string
    65  	cluster      string
    66  	retest       string
    67  }
    68  
    69  func (jl *jobLabel) values() []string {
    70  	return []string{jl.jobNamespace, jl.jobName, jl.jobType, jl.state, jl.org, jl.repo, jl.baseRef, jl.cluster, jl.retest}
    71  }
    72  
    73  func init() {
    74  	prometheus.MustRegister(prowJobs)
    75  	prometheus.MustRegister(prowJobTransitions)
    76  }
    77  
    78  func getJobLabelMap(pjs []prowapi.ProwJob) map[jobLabel]float64 {
    79  	jobLabelMap := make(map[jobLabel]float64)
    80  
    81  	for _, pj := range pjs {
    82  		jobLabelMap[getJobLabel(pj)]++
    83  	}
    84  	return jobLabelMap
    85  }
    86  
    87  func getJobLabel(pj prowapi.ProwJob) jobLabel {
    88  	jl := jobLabel{jobNamespace: pj.Namespace, jobName: pj.Spec.Job, jobType: string(pj.Spec.Type), state: string(pj.Status.State), cluster: pj.Spec.Cluster}
    89  
    90  	if pj.Spec.Refs != nil {
    91  		jl.org = pj.Spec.Refs.Org
    92  		jl.repo = pj.Spec.Refs.Repo
    93  		jl.baseRef = pj.Spec.Refs.BaseRef
    94  	} else if len(pj.Spec.ExtraRefs) > 0 {
    95  		jl.org = pj.Spec.ExtraRefs[0].Org
    96  		jl.repo = pj.Spec.ExtraRefs[0].Repo
    97  		jl.baseRef = pj.Spec.ExtraRefs[0].BaseRef
    98  	}
    99  
   100  	if retest, exists := pj.ObjectMeta.Labels[RetestLabel]; exists && retest == "true" {
   101  		jl.retest = "true"
   102  	} else {
   103  		jl.retest = "false"
   104  	}
   105  
   106  	return jl
   107  }
   108  
   109  type jobIdentifier struct {
   110  	jobLabel
   111  	buildId string
   112  }
   113  
   114  func getJobIdentifier(pj prowapi.ProwJob) jobIdentifier {
   115  	return jobIdentifier{
   116  		jobLabel: getJobLabel(pj),
   117  		buildId:  pj.Status.BuildID,
   118  	}
   119  }
   120  
   121  // previousStates records the prowJobs we were called with previously
   122  var previousStates map[jobIdentifier]prowapi.ProwJobState
   123  
   124  // GatherProwJobMetrics gathers prometheus metrics for prowjobs.
   125  // Not threadsafe, ensure this is called serially.
   126  func GatherProwJobMetrics(l *logrus.Entry, current []prowapi.ProwJob) {
   127  	// This may be racing with the prometheus server but we need to remove
   128  	// stale metrics like triggered or pending jobs that are now complete.
   129  	prowJobs.Reset()
   130  
   131  	// record the current state of ProwJob CRs on the system
   132  	for jl, count := range getJobLabelMap(current) {
   133  		prowJobs.WithLabelValues(jl.values()...).Set(count)
   134  	}
   135  
   136  	// record state transitions since the last time we were called
   137  	currentStates := map[jobIdentifier]prowapi.ProwJobState{}
   138  	for _, pj := range current {
   139  		ji := getJobIdentifier(pj)
   140  		state := prowapi.ProwJobState(ji.state)
   141  		currentStates[ji] = state
   142  
   143  		if previousState, seen := previousStates[ji]; !seen || previousState != state {
   144  			prowJobTransitions.WithLabelValues(ji.values()...).Inc()
   145  		}
   146  	}
   147  
   148  	previousStates = currentStates
   149  }