github.com/timstclair/heapster@v0.20.0-alpha1/metrics/sinks/manager.go (about)

     1  // Copyright 2015 Google Inc. All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package sinks
    16  
    17  import (
    18  	"sync"
    19  	"time"
    20  
    21  	"github.com/golang/glog"
    22  	"github.com/prometheus/client_golang/prometheus"
    23  	"k8s.io/heapster/metrics/core"
    24  )
    25  
    26  const (
    27  	DefaultSinkExportDataTimeout = 20 * time.Second
    28  	DefaultSinkStopTimeout       = 60 * time.Second
    29  )
    30  
    31  var (
    32  	// Last time Heapster exported data since unix epoch in seconds.
    33  	lastExportTimestamp = prometheus.NewGaugeVec(
    34  		prometheus.GaugeOpts{
    35  			Namespace: "heapster",
    36  			Subsystem: "exporter",
    37  			Name:      "last_time_seconds",
    38  			Help:      "Last time Heapster exported data since unix epoch in seconds.",
    39  		},
    40  		[]string{"exporter"},
    41  	)
    42  
    43  	// Time spent exporting data to sink in microseconds.
    44  	exporterDuration = prometheus.NewSummaryVec(
    45  		prometheus.SummaryOpts{
    46  			Namespace: "heapster",
    47  			Subsystem: "exporter",
    48  			Name:      "duration_microseconds",
    49  			Help:      "Time spent exporting data to sink in microseconds.",
    50  		},
    51  		[]string{"exporter"},
    52  	)
    53  )
    54  
    55  func init() {
    56  	prometheus.MustRegister(lastExportTimestamp)
    57  	prometheus.MustRegister(exporterDuration)
    58  }
    59  
    60  type sinkHolder struct {
    61  	sink             core.DataSink
    62  	dataBatchChannel chan *core.DataBatch
    63  	stopChannel      chan bool
    64  }
    65  
    66  // Sink Manager - a special sink that distributes data to other sinks. It pushes data
    67  // only to these sinks that completed their previous exports. Data that could not be
    68  // pushed in the defined time is dropped and not retried.
    69  type sinkManager struct {
    70  	sinkHolders       []sinkHolder
    71  	exportDataTimeout time.Duration
    72  	stopTimeout       time.Duration
    73  }
    74  
    75  func NewDataSinkManager(sinks []core.DataSink, exportDataTimeout, stopTimeout time.Duration) (core.DataSink, error) {
    76  	sinkHolders := []sinkHolder{}
    77  	for _, sink := range sinks {
    78  		sh := sinkHolder{
    79  			sink:             sink,
    80  			dataBatchChannel: make(chan *core.DataBatch),
    81  			stopChannel:      make(chan bool),
    82  		}
    83  		sinkHolders = append(sinkHolders, sh)
    84  		go func(sh sinkHolder) {
    85  			for {
    86  				select {
    87  				case data := <-sh.dataBatchChannel:
    88  					export(sh.sink, data)
    89  				case isStop := <-sh.stopChannel:
    90  					glog.V(2).Infof("Stop received: %s", sh.sink.Name())
    91  					if isStop {
    92  						sh.sink.Stop()
    93  						return
    94  					}
    95  				}
    96  			}
    97  		}(sh)
    98  	}
    99  	return &sinkManager{
   100  		sinkHolders:       sinkHolders,
   101  		exportDataTimeout: exportDataTimeout,
   102  		stopTimeout:       stopTimeout,
   103  	}, nil
   104  }
   105  
   106  // Guarantees that the export will complete in sinkExportDataTimeout.
   107  func (this *sinkManager) ExportData(data *core.DataBatch) {
   108  	var wg sync.WaitGroup
   109  	for _, sh := range this.sinkHolders {
   110  		wg.Add(1)
   111  		go func(sh sinkHolder, wg *sync.WaitGroup) {
   112  			defer wg.Done()
   113  			glog.V(2).Infof("Pushing data to: %s", sh.sink.Name())
   114  			select {
   115  			case sh.dataBatchChannel <- data:
   116  				glog.V(2).Infof("Data push completed: %s", sh.sink.Name())
   117  				// everything ok
   118  			case <-time.After(this.exportDataTimeout):
   119  				glog.Warningf("Failed to push data to sink: %s", sh.sink.Name())
   120  			}
   121  		}(sh, &wg)
   122  	}
   123  	// Wait for all pushes to complete or timeout.
   124  	wg.Wait()
   125  }
   126  
   127  func (this *sinkManager) Name() string {
   128  	return "Manager"
   129  }
   130  
   131  func (this *sinkManager) Stop() {
   132  	for _, sh := range this.sinkHolders {
   133  		glog.V(2).Infof("Running stop for: %s", sh.sink.Name())
   134  
   135  		go func(sh sinkHolder) {
   136  			select {
   137  			case sh.stopChannel <- true:
   138  				// everything ok
   139  				glog.V(2).Infof("Stop sent to sink: %s", sh.sink.Name())
   140  
   141  			case <-time.After(this.stopTimeout):
   142  				glog.Warningf("Failed to stop sink: %s", sh.sink.Name())
   143  			}
   144  			return
   145  		}(sh)
   146  	}
   147  }
   148  
   149  func export(s core.DataSink, data *core.DataBatch) {
   150  	startTime := time.Now()
   151  	defer lastExportTimestamp.
   152  		WithLabelValues(s.Name()).
   153  		Set(float64(time.Now().Unix()))
   154  	defer exporterDuration.
   155  		WithLabelValues(s.Name()).
   156  		Observe(float64(time.Since(startTime)) / float64(time.Microsecond))
   157  
   158  	s.ExportData(data)
   159  }