github.com/jonaz/heapster@v1.3.0-beta.0.0.20170208112634-cd3c15ca3d29/metrics/sinks/influxdb/influxdb_test.go (about)

     1  // Copyright 2015 Google Inc. All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package influxdb
    16  
    17  import (
    18  	"encoding/json"
    19  	"fmt"
    20  	"testing"
    21  	"time"
    22  
    23  	"net/http/httptest"
    24  	"net/url"
    25  
    26  	influx_models "github.com/influxdata/influxdb/models"
    27  	"github.com/stretchr/testify/assert"
    28  	influxdb_common "k8s.io/heapster/common/influxdb"
    29  	"k8s.io/heapster/metrics/core"
    30  	util "k8s.io/kubernetes/pkg/util/testing"
    31  )
    32  
    33  type fakeInfluxDBDataSink struct {
    34  	core.DataSink
    35  	fakeDbClient *influxdb_common.FakeInfluxDBClient
    36  }
    37  
    38  func newRawInfluxSink() *influxdbSink {
    39  	return &influxdbSink{
    40  		client: influxdb_common.Client,
    41  		c:      influxdb_common.Config,
    42  	}
    43  }
    44  
    45  func NewFakeSink() fakeInfluxDBDataSink {
    46  	return fakeInfluxDBDataSink{
    47  		newRawInfluxSink(),
    48  		influxdb_common.Client,
    49  	}
    50  }
    51  func TestStoreDataEmptyInput(t *testing.T) {
    52  	fakeSink := NewFakeSink()
    53  	dataBatch := core.DataBatch{}
    54  	fakeSink.ExportData(&dataBatch)
    55  	assert.Equal(t, 0, len(fakeSink.fakeDbClient.Pnts))
    56  }
    57  
    58  func TestStoreMultipleDataInput(t *testing.T) {
    59  	fakeSink := NewFakeSink()
    60  	timestamp := time.Now()
    61  
    62  	l := make(map[string]string)
    63  	l["namespace_id"] = "123"
    64  	l["container_name"] = "/system.slice/-.mount"
    65  	l[core.LabelPodId.Key] = "aaaa-bbbb-cccc-dddd"
    66  
    67  	l2 := make(map[string]string)
    68  	l2["namespace_id"] = "123"
    69  	l2["container_name"] = "/system.slice/dbus.service"
    70  	l2[core.LabelPodId.Key] = "aaaa-bbbb-cccc-dddd"
    71  
    72  	l3 := make(map[string]string)
    73  	l3["namespace_id"] = "123"
    74  	l3[core.LabelPodId.Key] = "aaaa-bbbb-cccc-dddd"
    75  
    76  	l4 := make(map[string]string)
    77  	l4["namespace_id"] = ""
    78  	l4[core.LabelPodId.Key] = "aaaa-bbbb-cccc-dddd"
    79  
    80  	l5 := make(map[string]string)
    81  	l5["namespace_id"] = "123"
    82  	l5[core.LabelPodId.Key] = "aaaa-bbbb-cccc-dddd"
    83  
    84  	metricSet1 := core.MetricSet{
    85  		Labels: l,
    86  		MetricValues: map[string]core.MetricValue{
    87  			"/system.slice/-.mount//cpu/limit": {
    88  				ValueType:  core.ValueInt64,
    89  				MetricType: core.MetricCumulative,
    90  				IntValue:   123456,
    91  			},
    92  		},
    93  	}
    94  
    95  	metricSet2 := core.MetricSet{
    96  		Labels: l2,
    97  		MetricValues: map[string]core.MetricValue{
    98  			"/system.slice/dbus.service//cpu/usage": {
    99  				ValueType:  core.ValueInt64,
   100  				MetricType: core.MetricCumulative,
   101  				IntValue:   123456,
   102  			},
   103  		},
   104  	}
   105  
   106  	metricSet3 := core.MetricSet{
   107  		Labels: l3,
   108  		MetricValues: map[string]core.MetricValue{
   109  			"test/metric/1": {
   110  				ValueType:  core.ValueInt64,
   111  				MetricType: core.MetricCumulative,
   112  				IntValue:   123456,
   113  			},
   114  		},
   115  	}
   116  
   117  	metricSet4 := core.MetricSet{
   118  		Labels: l4,
   119  		MetricValues: map[string]core.MetricValue{
   120  			"test/metric/1": {
   121  				ValueType:  core.ValueInt64,
   122  				MetricType: core.MetricCumulative,
   123  				IntValue:   123456,
   124  			},
   125  		},
   126  	}
   127  
   128  	metricSet5 := core.MetricSet{
   129  		Labels: l5,
   130  		MetricValues: map[string]core.MetricValue{
   131  			"removeme": {
   132  				ValueType:  core.ValueInt64,
   133  				MetricType: core.MetricCumulative,
   134  				IntValue:   123456,
   135  			},
   136  		},
   137  	}
   138  
   139  	data := core.DataBatch{
   140  		Timestamp: timestamp,
   141  		MetricSets: map[string]*core.MetricSet{
   142  			"pod1": &metricSet1,
   143  			"pod2": &metricSet2,
   144  			"pod3": &metricSet3,
   145  			"pod4": &metricSet4,
   146  			"pod5": &metricSet5,
   147  		},
   148  	}
   149  
   150  	fakeSink.ExportData(&data)
   151  	assert.Equal(t, 5, len(fakeSink.fakeDbClient.Pnts))
   152  }
   153  
   154  func TestCreateInfluxdbSink(t *testing.T) {
   155  	handler := util.FakeHandler{
   156  		StatusCode:   200,
   157  		RequestBody:  "",
   158  		ResponseBody: "",
   159  		T:            t,
   160  	}
   161  	server := httptest.NewServer(&handler)
   162  	defer server.Close()
   163  
   164  	stubInfluxDBUrl, err := url.Parse(server.URL)
   165  	assert.NoError(t, err)
   166  
   167  	//create influxdb sink
   168  	sink, err := CreateInfluxdbSink(stubInfluxDBUrl)
   169  	assert.NoError(t, err)
   170  
   171  	//check sink name
   172  	assert.Equal(t, sink.Name(), "InfluxDB Sink")
   173  }
   174  
   175  func makeRow(results [][]string) influx_models.Row {
   176  	resRow := influx_models.Row{
   177  		Values: make([][]interface{}, len(results)),
   178  	}
   179  
   180  	for setInd, valSet := range results {
   181  		outVals := make([]interface{}, len(valSet))
   182  		for valInd, val := range valSet {
   183  			if valInd == 0 {
   184  				// timestamp should just be a string
   185  				outVals[valInd] = val
   186  			} else {
   187  				outVals[valInd] = json.Number(val)
   188  			}
   189  		}
   190  		resRow.Values[setInd] = outVals
   191  	}
   192  
   193  	return resRow
   194  }
   195  
   196  func checkMetricVal(expected, actual core.MetricValue) bool {
   197  	if expected.ValueType != actual.ValueType {
   198  		return false
   199  	}
   200  
   201  	// only check the relevant value type
   202  	switch expected.ValueType {
   203  	case core.ValueFloat:
   204  		return expected.FloatValue == actual.FloatValue
   205  	case core.ValueInt64:
   206  		return expected.IntValue == actual.IntValue
   207  	default:
   208  		return expected == actual
   209  	}
   210  }
   211  
   212  func TestHistoricalMissingResponses(t *testing.T) {
   213  	sink := newRawInfluxSink()
   214  
   215  	podKeys := []core.HistoricalKey{
   216  		{ObjectType: core.MetricSetTypePod, NamespaceName: "cheese", PodName: "cheddar"},
   217  		{ObjectType: core.MetricSetTypePod, NamespaceName: "cheese", PodName: "swiss"},
   218  	}
   219  	labels := map[string]string{"crackers": "ritz"}
   220  
   221  	errStr := fmt.Sprintf("No results for metric %q describing %q", "cpu/usage_rate", podKeys[0].String())
   222  
   223  	_, err := sink.GetMetric("cpu/usage_rate", podKeys, time.Now().Add(-5*time.Minute), time.Now())
   224  	assert.EqualError(t, err, errStr)
   225  
   226  	_, err = sink.GetLabeledMetric("cpu/usage_rate", labels, podKeys, time.Now().Add(-5*time.Minute), time.Now())
   227  	assert.EqualError(t, err, errStr)
   228  
   229  	_, err = sink.GetAggregation("cpu/usage_rate", []core.AggregationType{core.AggregationTypeAverage}, podKeys, time.Now().Add(-5*time.Minute), time.Now(), 5*time.Minute)
   230  	assert.EqualError(t, err, errStr)
   231  
   232  	_, err = sink.GetLabeledAggregation("cpu/usage_rate", labels, []core.AggregationType{core.AggregationTypeAverage}, podKeys, time.Now().Add(-5*time.Minute), time.Now(), 5*time.Minute)
   233  	assert.EqualError(t, err, errStr)
   234  }
   235  
   236  func TestHistoricalInfluxRawMetricsParsing(t *testing.T) {
   237  	// in order to just test the parsing, we just go directly to the sink type
   238  	sink := newRawInfluxSink()
   239  
   240  	baseTime := time.Time{}
   241  
   242  	rawTests := []struct {
   243  		name            string
   244  		rawData         influx_models.Row
   245  		expectedResults []core.TimestampedMetricValue
   246  		expectedError   bool
   247  	}{
   248  		{
   249  			name: "all-integer data",
   250  			rawData: makeRow([][]string{
   251  				{
   252  					baseTime.Add(24 * time.Hour).Format(time.RFC3339),
   253  					"1234",
   254  				},
   255  				{
   256  					baseTime.Add(48 * time.Hour).Format(time.RFC3339),
   257  					"5678",
   258  				},
   259  			}),
   260  			expectedResults: []core.TimestampedMetricValue{
   261  				{
   262  					Timestamp:   baseTime.Add(24 * time.Hour),
   263  					MetricValue: core.MetricValue{IntValue: 1234, ValueType: core.ValueInt64},
   264  				},
   265  				{
   266  					Timestamp:   baseTime.Add(48 * time.Hour),
   267  					MetricValue: core.MetricValue{IntValue: 5678, ValueType: core.ValueInt64},
   268  				},
   269  			},
   270  		},
   271  		{
   272  			name: "all-float data",
   273  			rawData: makeRow([][]string{
   274  				{
   275  					baseTime.Add(24 * time.Hour).Format(time.RFC3339),
   276  					"1.23e10",
   277  				},
   278  				{
   279  					baseTime.Add(48 * time.Hour).Format(time.RFC3339),
   280  					"4.56e11",
   281  				},
   282  			}),
   283  			expectedResults: []core.TimestampedMetricValue{
   284  				{
   285  					Timestamp:   baseTime.Add(24 * time.Hour),
   286  					MetricValue: core.MetricValue{FloatValue: 12300000000.0, ValueType: core.ValueFloat},
   287  				},
   288  				{
   289  					Timestamp:   baseTime.Add(48 * time.Hour),
   290  					MetricValue: core.MetricValue{FloatValue: 456000000000.0, ValueType: core.ValueFloat},
   291  				},
   292  			},
   293  		},
   294  		{
   295  			name: "mixed data",
   296  			rawData: makeRow([][]string{
   297  				{
   298  					baseTime.Add(24 * time.Hour).Format(time.RFC3339),
   299  					"123",
   300  				},
   301  				{
   302  					baseTime.Add(48 * time.Hour).Format(time.RFC3339),
   303  					"4.56e11",
   304  				},
   305  			}),
   306  			expectedResults: []core.TimestampedMetricValue{
   307  				{
   308  					Timestamp:   baseTime.Add(24 * time.Hour),
   309  					MetricValue: core.MetricValue{FloatValue: 123.0, ValueType: core.ValueFloat},
   310  				},
   311  				{
   312  					Timestamp:   baseTime.Add(48 * time.Hour),
   313  					MetricValue: core.MetricValue{FloatValue: 456000000000.0, ValueType: core.ValueFloat},
   314  				},
   315  			},
   316  		},
   317  		{
   318  			name: "data with invalid value",
   319  			rawData: makeRow([][]string{
   320  				{
   321  					baseTime.Add(24 * time.Hour).Format(time.RFC3339),
   322  					"true",
   323  				},
   324  			}),
   325  			expectedError: true,
   326  		},
   327  	}
   328  
   329  RAWTESTLOOP:
   330  	for _, test := range rawTests {
   331  		parsedRawResults, err := sink.parseRawQueryRow(test.rawData)
   332  		if (err != nil) != test.expectedError {
   333  			t.Errorf("When parsing raw %s: expected error %v != actual error %v", test.name, test.expectedError, err)
   334  			continue RAWTESTLOOP
   335  		}
   336  
   337  		if len(parsedRawResults) != len(test.expectedResults) {
   338  			t.Errorf("When parsing raw %s: expected results %#v != actual results %#v", test.name, test.expectedResults, parsedRawResults)
   339  			continue RAWTESTLOOP
   340  		}
   341  
   342  		for i, metricVal := range parsedRawResults {
   343  			if !test.expectedResults[i].Timestamp.Equal(metricVal.Timestamp) {
   344  				t.Errorf("When parsing raw %s: expected results %#v != actual results %#v", test.name, test.expectedResults, parsedRawResults)
   345  				continue RAWTESTLOOP
   346  			}
   347  
   348  			if !checkMetricVal(test.expectedResults[i].MetricValue, metricVal.MetricValue) {
   349  				t.Errorf("When parsing raw %s: expected results %#v != actual results %#v", test.name, test.expectedResults, parsedRawResults)
   350  				continue RAWTESTLOOP
   351  			}
   352  		}
   353  	}
   354  
   355  	var countVal2 uint64 = 2
   356  	aggregatedTests := []struct {
   357  		name            string
   358  		rawData         influx_models.Row
   359  		expectedResults []core.TimestampedAggregationValue
   360  		expectedError   bool
   361  	}{
   362  		{
   363  			name: "all-integer data",
   364  			rawData: makeRow([][]string{
   365  				{
   366  					baseTime.Add(24 * time.Hour).Format(time.RFC3339),
   367  					"2",
   368  					"1234",
   369  				},
   370  				{
   371  					baseTime.Add(48 * time.Hour).Format(time.RFC3339),
   372  					"2",
   373  					"5678",
   374  				},
   375  			}),
   376  			expectedResults: []core.TimestampedAggregationValue{
   377  				{
   378  					Timestamp: baseTime.Add(24 * time.Hour),
   379  					AggregationValue: core.AggregationValue{
   380  						Count: &countVal2,
   381  						Aggregations: map[core.AggregationType]core.MetricValue{
   382  							core.AggregationTypeAverage: {IntValue: 1234, ValueType: core.ValueInt64},
   383  						},
   384  					},
   385  				},
   386  				{
   387  					Timestamp: baseTime.Add(48 * time.Hour),
   388  					AggregationValue: core.AggregationValue{
   389  						Count: &countVal2,
   390  						Aggregations: map[core.AggregationType]core.MetricValue{
   391  							core.AggregationTypeAverage: {IntValue: 5678, ValueType: core.ValueInt64},
   392  						},
   393  					},
   394  				},
   395  			},
   396  		},
   397  		{
   398  			name: "all-float data",
   399  			rawData: makeRow([][]string{
   400  				{
   401  					baseTime.Add(24 * time.Hour).Format(time.RFC3339),
   402  					"2",
   403  					"1.23e10",
   404  				},
   405  				{
   406  					baseTime.Add(48 * time.Hour).Format(time.RFC3339),
   407  					"2",
   408  					"4.56e11",
   409  				},
   410  			}),
   411  			expectedResults: []core.TimestampedAggregationValue{
   412  				{
   413  					Timestamp: baseTime.Add(24 * time.Hour),
   414  					AggregationValue: core.AggregationValue{
   415  						Count: &countVal2,
   416  						Aggregations: map[core.AggregationType]core.MetricValue{
   417  							core.AggregationTypeAverage: {FloatValue: 12300000000.0, ValueType: core.ValueFloat},
   418  						},
   419  					},
   420  				},
   421  				{
   422  					Timestamp: baseTime.Add(48 * time.Hour),
   423  					AggregationValue: core.AggregationValue{
   424  						Count: &countVal2,
   425  						Aggregations: map[core.AggregationType]core.MetricValue{
   426  							core.AggregationTypeAverage: {FloatValue: 456000000000.0, ValueType: core.ValueFloat},
   427  						},
   428  					},
   429  				},
   430  			},
   431  		},
   432  		{
   433  			name: "mixed data",
   434  			rawData: makeRow([][]string{
   435  				{
   436  					baseTime.Add(24 * time.Hour).Format(time.RFC3339),
   437  					"2",
   438  					"123",
   439  				},
   440  				{
   441  					baseTime.Add(48 * time.Hour).Format(time.RFC3339),
   442  					"2",
   443  					"4.56e11",
   444  				},
   445  			}),
   446  			expectedResults: []core.TimestampedAggregationValue{
   447  				{
   448  					Timestamp: baseTime.Add(24 * time.Hour),
   449  					AggregationValue: core.AggregationValue{
   450  						Count: &countVal2,
   451  						Aggregations: map[core.AggregationType]core.MetricValue{
   452  							core.AggregationTypeAverage: {FloatValue: 123.0, ValueType: core.ValueFloat},
   453  						},
   454  					},
   455  				},
   456  				{
   457  					Timestamp: baseTime.Add(48 * time.Hour),
   458  					AggregationValue: core.AggregationValue{
   459  						Count: &countVal2,
   460  						Aggregations: map[core.AggregationType]core.MetricValue{
   461  							core.AggregationTypeAverage: {FloatValue: 456000000000.0, ValueType: core.ValueFloat},
   462  						},
   463  					},
   464  				},
   465  			},
   466  		},
   467  		{
   468  			name: "data with invalid value",
   469  			rawData: makeRow([][]string{
   470  				{
   471  					baseTime.Add(24 * time.Hour).Format(time.RFC3339),
   472  					"2",
   473  					"true",
   474  				},
   475  			}),
   476  			expectedError: true,
   477  		},
   478  	}
   479  
   480  	aggregationLookup := map[core.AggregationType]int{
   481  		core.AggregationTypeCount:   1,
   482  		core.AggregationTypeAverage: 2,
   483  	}
   484  AGGTESTLOOP:
   485  	for _, test := range aggregatedTests {
   486  		parsedAggResults, err := sink.parseAggregateQueryRow(test.rawData, aggregationLookup, 24*time.Hour)
   487  		if (err != nil) != test.expectedError {
   488  			t.Errorf("When parsing aggregated %s: expected error %v != actual error %v", test.name, test.expectedError, err)
   489  			continue AGGTESTLOOP
   490  		}
   491  
   492  		if len(parsedAggResults) != len(test.expectedResults) {
   493  			t.Errorf("When parsing aggregated %s: expected results %#v had a different length from actual results %#v", test.name, test.expectedResults, parsedAggResults)
   494  			continue AGGTESTLOOP
   495  		}
   496  
   497  		for i, metricVal := range parsedAggResults {
   498  			expVal := test.expectedResults[i]
   499  			if !expVal.Timestamp.Equal(metricVal.Timestamp) {
   500  				t.Errorf("When parsing aggregated %s: expected results %#v had a different timestamp from actual results %#v", test.name, expVal, metricVal)
   501  				continue AGGTESTLOOP
   502  			}
   503  
   504  			if len(expVal.Aggregations) != len(metricVal.Aggregations) {
   505  				t.Errorf("When parsing aggregated %s: expected results %#v had a number of aggregations from actual results %#v", test.name, expVal, metricVal)
   506  				continue AGGTESTLOOP
   507  			}
   508  
   509  			for aggName, aggVal := range metricVal.Aggregations {
   510  				if expAggVal, ok := expVal.Aggregations[aggName]; !ok || !checkMetricVal(expAggVal, aggVal) {
   511  					t.Errorf("When parsing aggregated %s: expected results %#v != actual results %#v", test.name, expAggVal, aggVal)
   512  					continue AGGTESTLOOP
   513  				}
   514  			}
   515  		}
   516  	}
   517  }
   518  
   519  func TestSanitizers(t *testing.T) {
   520  	badMetricName := "foo; baz"
   521  	goodMetricName := "cheese/types-crackers"
   522  
   523  	goodKeyValue := "cheddar.CHEESE-ritz.Crackers_1"
   524  	badKeyValue := "foobar'; baz"
   525  
   526  	sink := newRawInfluxSink()
   527  
   528  	if err := sink.checkSanitizedMetricName(goodMetricName); err != nil {
   529  		t.Errorf("Expected %q to be a valid metric name, but it was not: %v", goodMetricName, err)
   530  	}
   531  
   532  	if err := sink.checkSanitizedMetricName(badMetricName); err == nil {
   533  		t.Errorf("Expected %q to be an invalid metric name, but it was valid", badMetricName)
   534  	}
   535  
   536  	badKeys := []core.HistoricalKey{
   537  		{
   538  			NodeName: badKeyValue,
   539  		},
   540  		{
   541  			NamespaceName: badKeyValue,
   542  		},
   543  		{
   544  			PodName: badKeyValue,
   545  		},
   546  		{
   547  			ContainerName: badKeyValue,
   548  		},
   549  		{
   550  			PodId: badKeyValue,
   551  		},
   552  	}
   553  
   554  	for _, key := range badKeys {
   555  		if err := sink.checkSanitizedKey(&key); err == nil {
   556  			t.Errorf("Expected key %#v to be invalid, but it was not", key)
   557  		}
   558  	}
   559  
   560  	goodKeys := []core.HistoricalKey{
   561  		{
   562  			NodeName: goodKeyValue,
   563  		},
   564  		{
   565  			NamespaceName: goodKeyValue,
   566  		},
   567  		{
   568  			PodName: goodKeyValue,
   569  		},
   570  		{
   571  			ContainerName: goodKeyValue,
   572  		},
   573  		{
   574  			PodId: goodKeyValue,
   575  		},
   576  	}
   577  
   578  	for _, key := range goodKeys {
   579  		if err := sink.checkSanitizedKey(&key); err != nil {
   580  			t.Errorf("Expected key %#v to be valid, but it was not: %v", key, err)
   581  		}
   582  	}
   583  }