github.com/galamsiva2020/kubernetes-heapster-monitoring@v0.0.0-20210823134957-3c1baa7c1e70/metrics/manager/manager.go (about) 1 // Copyright 2015 Google Inc. All Rights Reserved. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package manager 16 17 import ( 18 "time" 19 20 "k8s.io/heapster/metrics/core" 21 22 "github.com/golang/glog" 23 "github.com/prometheus/client_golang/prometheus" 24 ) 25 26 const ( 27 DefaultScrapeOffset = 5 * time.Second 28 DefaultMaxParallelism = 3 29 ) 30 31 var ( 32 // The Time spent in a processor in milliseconds. 33 processorDuration = prometheus.NewSummaryVec( 34 prometheus.SummaryOpts{ 35 Namespace: "heapster", 36 Subsystem: "processor", 37 Name: "duration_milliseconds", 38 Help: "The Time spent in a processor in milliseconds.", 39 }, 40 []string{"processor"}, 41 ) 42 ) 43 44 func init() { 45 prometheus.MustRegister(processorDuration) 46 } 47 48 type Manager interface { 49 Start() 50 Stop() 51 } 52 53 type realManager struct { 54 source core.MetricsSource 55 processors []core.DataProcessor 56 sink core.DataSink 57 resolution time.Duration 58 scrapeOffset time.Duration 59 stopChan chan struct{} 60 housekeepSemaphoreChan chan struct{} 61 housekeepTimeout time.Duration 62 } 63 64 func NewManager(source core.MetricsSource, processors []core.DataProcessor, sink core.DataSink, resolution time.Duration, 65 scrapeOffset time.Duration, maxParallelism int) (Manager, error) { 66 manager := realManager{ 67 source: source, 68 processors: processors, 69 sink: sink, 70 resolution: resolution, 71 scrapeOffset: scrapeOffset, 72 stopChan: make(chan struct{}), 73 housekeepSemaphoreChan: make(chan struct{}, maxParallelism), 74 housekeepTimeout: resolution / 2, 75 } 76 77 for i := 0; i < maxParallelism; i++ { 78 manager.housekeepSemaphoreChan <- struct{}{} 79 } 80 81 return &manager, nil 82 } 83 84 func (rm *realManager) Start() { 85 go rm.Housekeep() 86 } 87 88 func (rm *realManager) Stop() { 89 rm.stopChan <- struct{}{} 90 } 91 92 func (rm *realManager) Housekeep() { 93 for { 94 // Always try to get the newest metrics 95 now := time.Now() 96 start := now.Truncate(rm.resolution) 97 end := start.Add(rm.resolution) 98 timeToNextSync := end.Add(rm.scrapeOffset).Sub(now) 99 100 select { 101 case <-time.After(timeToNextSync): 102 rm.housekeep(start, end) 103 case <-rm.stopChan: 104 rm.sink.Stop() 105 return 106 } 107 } 108 } 109 110 func (rm *realManager) housekeep(start, end time.Time) { 111 if !start.Before(end) { 112 glog.Warningf("Wrong time provided to housekeep start:%s end: %s", start, end) 113 return 114 } 115 116 select { 117 case <-rm.housekeepSemaphoreChan: 118 // ok, good to go 119 120 case <-time.After(rm.housekeepTimeout): 121 glog.Warningf("Spent too long waiting for housekeeping to start") 122 return 123 } 124 125 go func(rm *realManager) { 126 // should always give back the semaphore 127 defer func() { rm.housekeepSemaphoreChan <- struct{}{} }() 128 data, err := rm.source.ScrapeMetrics(start, end) 129 130 if err != nil { 131 glog.Errorf("Error in scraping metrics for %s: %v", rm.source.Name(), err) 132 return 133 } 134 135 for _, p := range rm.processors { 136 newData, err := process(p, data) 137 if err == nil { 138 data = newData 139 } else { 140 glog.Errorf("Error in processor: %v", err) 141 return 142 } 143 } 144 145 // Export data to sinks 146 rm.sink.ExportData(data) 147 }(rm) 148 } 149 150 func process(p core.DataProcessor, data *core.DataBatch) (*core.DataBatch, error) { 151 startTime := time.Now() 152 defer func() { 153 processorDuration. 154 WithLabelValues(p.Name()). 155 Observe(float64(time.Since(startTime)) / float64(time.Millisecond)) 156 }() 157 158 return p.Process(data) 159 }