github.com/timstclair/heapster@v0.20.0-alpha1/metrics/sinks/kafka/driver.go (about)

     1  // Copyright 2015 Google Inc. All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package kafka
    16  
    17  import (
    18  	"encoding/json"
    19  	"fmt"
    20  	"net/url"
    21  	"sync"
    22  	"time"
    23  
    24  	"github.com/golang/glog"
    25  	"github.com/optiopay/kafka"
    26  	"github.com/optiopay/kafka/proto"
    27  	"k8s.io/heapster/metrics/core"
    28  )
    29  
    30  const (
    31  	partition                = 0
    32  	brokerClientID           = "kafka-sink"
    33  	brokerDialTimeout        = 10 * time.Second
    34  	brokerDialRetryLimit     = 1
    35  	brokerDialRetryWait      = 0
    36  	brokerAllowTopicCreation = true
    37  	brokerLeaderRetryLimit   = 1
    38  	brokerLeaderRetryWait    = 0
    39  	dataTopic                = "heapster-metrics"
    40  )
    41  
    42  type KafkaSinkPoint struct {
    43  	MetricsName      string
    44  	MetricsValue     interface{}
    45  	MetricsTimestamp time.Time
    46  	MetricsTags      map[string]string
    47  }
    48  
    49  type kafkaSink struct {
    50  	producer  kafka.Producer
    51  	dataTopic string
    52  	sync.RWMutex
    53  }
    54  
    55  func (sink *kafkaSink) ExportData(dataBatch *core.DataBatch) {
    56  	sink.Lock()
    57  	defer sink.Unlock()
    58  
    59  	for _, metricSet := range dataBatch.MetricSets {
    60  		for metricName, metricValue := range metricSet.MetricValues {
    61  
    62  			var value interface{}
    63  			if core.ValueInt64 == metricValue.ValueType {
    64  				value = metricValue.IntValue
    65  			} else if core.ValueFloat == metricValue.ValueType {
    66  				value = metricValue.FloatValue
    67  			} else {
    68  				continue
    69  			}
    70  
    71  			point := KafkaSinkPoint{
    72  				MetricsName: metricName,
    73  				MetricsTags: metricSet.Labels,
    74  				MetricsValue: map[string]interface{}{
    75  					"value": value,
    76  				},
    77  				MetricsTimestamp: dataBatch.Timestamp.UTC(),
    78  			}
    79  			sink.produceKafkaMessage(point, sink.dataTopic)
    80  		}
    81  	}
    82  }
    83  
    84  func (sink *kafkaSink) produceKafkaMessage(dataPoint KafkaSinkPoint, topic string) error {
    85  	start := time.Now()
    86  	jsonItems, err := json.Marshal(dataPoint)
    87  	if err != nil {
    88  		return fmt.Errorf("failed to transform the items to json : %s", err)
    89  	}
    90  	message := &proto.Message{Value: []byte(string(jsonItems))}
    91  	_, err = sink.producer.Produce(topic, partition, message)
    92  	if err != nil {
    93  		return fmt.Errorf("failed to produce message to %s:%d: %s", topic, partition, err)
    94  	}
    95  	end := time.Now()
    96  	glog.V(4).Info("Exported %d data to kafka in %s", len([]byte(string(jsonItems))), end.Sub(start))
    97  	return nil
    98  }
    99  
   100  func (sink *kafkaSink) Name() string {
   101  	return "Apache Kafka Sink"
   102  }
   103  
   104  func (sink *kafkaSink) Stop() {
   105  	// nothing needs to be done.
   106  }
   107  
   108  // setupProducer returns a producer of kafka server
   109  func setupProducer(sinkBrokerHosts []string, brokerConf kafka.BrokerConf) (kafka.Producer, error) {
   110  	glog.V(3).Infof("attempting to setup kafka sink")
   111  	broker, err := kafka.Dial(sinkBrokerHosts, brokerConf)
   112  	if err != nil {
   113  		return nil, fmt.Errorf("failed to connect to kafka cluster: %s", err)
   114  	}
   115  	defer broker.Close()
   116  
   117  	//create kafka producer
   118  	conf := kafka.NewProducerConf()
   119  	conf.RequiredAcks = proto.RequiredAcksLocal
   120  	sinkProducer := broker.Producer(conf)
   121  	glog.V(3).Infof("kafka sink setup successfully")
   122  	return sinkProducer, nil
   123  }
   124  
   125  func NewKafkaSink(uri *url.URL) (core.DataSink, error) {
   126  	opts, err := url.ParseQuery(uri.RawQuery)
   127  	if err != nil {
   128  		return nil, fmt.Errorf("failed to parser url's query string: %s", err)
   129  	}
   130  
   131  	var topic string = dataTopic
   132  	if len(opts["timeseriestopic"]) > 0 {
   133  		topic = opts["timeseriestopic"][0]
   134  	}
   135  
   136  	var kafkaBrokers []string
   137  	if len(opts["brokers"]) < 1 {
   138  		return nil, fmt.Errorf("There is no broker assigned for connecting kafka")
   139  	}
   140  	kafkaBrokers = append(kafkaBrokers, opts["brokers"]...)
   141  	glog.V(2).Infof("initializing kafka sink with brokers - %v", kafkaBrokers)
   142  
   143  	//structure the config of broker
   144  	brokerConf := kafka.NewBrokerConf(brokerClientID)
   145  	brokerConf.DialTimeout = brokerDialTimeout
   146  	brokerConf.DialRetryLimit = brokerDialRetryLimit
   147  	brokerConf.DialRetryWait = brokerDialRetryWait
   148  	brokerConf.LeaderRetryLimit = brokerLeaderRetryLimit
   149  	brokerConf.LeaderRetryWait = brokerLeaderRetryWait
   150  	brokerConf.AllowTopicCreation = true
   151  
   152  	// set up producer of kafka server.
   153  	sinkProducer, err := setupProducer(kafkaBrokers, brokerConf)
   154  	if err != nil {
   155  		return nil, fmt.Errorf("Failed to setup Producer: - %v", err)
   156  	}
   157  
   158  	return &kafkaSink{
   159  		producer:  sinkProducer,
   160  		dataTopic: topic,
   161  	}, nil
   162  }