github.com/eth-easl/loader@v0.0.0-20230908084258-8a37e1d94279/pkg/metric/scrape_kn.py (about) 1 # MIT License 2 # 3 # Copyright (c) 2023 EASL and the vHive community 4 # 5 # Permission is hereby granted, free of charge, to any person obtaining a copy 6 # of this software and associated documentation files (the "Software"), to deal 7 # in the Software without restriction, including without limitation the rights 8 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 # copies of the Software, and to permit persons to whom the Software is 10 # furnished to do so, subject to the following conditions: 11 # 12 # The above copyright notice and this permission notice shall be included in all 13 # copies or substantial portions of the Software. 14 # 15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 # SOFTWARE. 22 23 import os 24 import json 25 26 prometheus_ip = os.popen("kubectl get svc -n monitoring | grep prometheus-kube-prometheus-prometheus | awk '{print $3}'").read().strip().split('\n')[0] 27 28 def get_promql_query(query): 29 def promql_query(): 30 return "tools/bin/promql --no-headers --host 'http://" + prometheus_ip + ":9090' '" + query + "' | awk '{print $1}'" 31 return promql_query 32 33 if __name__ == "__main__": 34 kn_status = { 35 # Desired counts set by autoscalers. 36 "desired_pods": get_promql_query('sum(autoscaler_desired_pods)'), 37 # Creating containers. 38 "unready_pods": get_promql_query('sum(autoscaler_not_ready_pods)'), 39 # Scheduling + image pulling. 40 "pending_pods": get_promql_query('sum(autoscaler_pending_pods)'), 41 # Number of pods autoscalers requested from Kubernetes. 42 "requested_pods": get_promql_query('sum(autoscaler_requested_pods)'), 43 "running_pods": get_promql_query('sum(autoscaler_actual_pods)'), 44 "activator_request_count": get_promql_query('sum(activator_request_count)'), 45 46 "autoscaler_stable_queue": get_promql_query('avg(autoscaler_stable_request_concurrency)'), 47 "autoscaler_panic_queue": get_promql_query('avg(autoscaler_panic_request_concurrency)'), 48 "activator_queue": get_promql_query('avg(activator_request_concurrency)'), 49 50 # The p95 latency of single scheduling round (algorithm+binding) over a time window of 30s. 51 "scheduling_p95": get_promql_query( 52 'histogram_quantile(0.95, sum by (le) (rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[30s])))' 53 ), 54 "scheduling_p50": get_promql_query( 55 'histogram_quantile(0.50, sum by (le) (rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[30s])))' 56 ), 57 58 # The p95 latency of E2E pod placement (potentially multiple scheduling rounds) over a time window of 30s. 59 "e2e_placement_p95": get_promql_query( 60 'histogram_quantile(0.95, sum by (le) (rate(scheduler_pod_scheduling_duration_seconds_bucket{job="kube-scheduler"}[30s])))' 61 ), 62 "e2e_placement_p50": get_promql_query( 63 'histogram_quantile(0.50, sum by (le) (rate(scheduler_pod_scheduling_duration_seconds_bucket{job="kube-scheduler"}[30s])))' 64 ), 65 } 66 67 for label, query in kn_status.items(): 68 69 while True: 70 try: 71 measure = os.popen(query()).read().strip() 72 if 'error' not in measure: 73 break 74 except: 75 pass 76 77 if label.endswith('queue'): 78 measure = float(measure) if measure else -99 79 elif 'p50' in label or 'p95' in label: 80 if measure == 'NaN': 81 # Not available. 82 measure = -99 83 else: 84 measure = float(measure) if measure else -99 85 else: 86 measure = int(measure) if measure else -99 87 88 kn_status[label] = measure 89 90 print(json.dumps(kn_status)) 91 92