github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/tools/querytee/response_comparator_test.go (about)

     1  package querytee
     2  
     3  import (
     4  	"encoding/json"
     5  	"errors"
     6  	"testing"
     7  	"time"
     8  
     9  	"github.com/prometheus/common/model"
    10  	"github.com/stretchr/testify/require"
    11  )
    12  
    13  func TestCompareMatrix(t *testing.T) {
    14  	for _, tc := range []struct {
    15  		name     string
    16  		expected json.RawMessage
    17  		actual   json.RawMessage
    18  		err      error
    19  	}{
    20  		{
    21  			name:     "no metrics",
    22  			expected: json.RawMessage(`[]`),
    23  			actual:   json.RawMessage(`[]`),
    24  		},
    25  		{
    26  			name: "no metrics in actual response",
    27  			expected: json.RawMessage(`[
    28  							{"metric":{"foo":"bar"},"values":[[1,"1"]]}
    29  						]`),
    30  			actual: json.RawMessage(`[]`),
    31  			err:    errors.New("expected 1 metrics but got 0"),
    32  		},
    33  		{
    34  			name: "extra metric in actual response",
    35  			expected: json.RawMessage(`[
    36  							{"metric":{"foo":"bar"},"values":[[1,"1"]]}
    37  						]`),
    38  			actual: json.RawMessage(`[
    39  							{"metric":{"foo":"bar"},"values":[[1,"1"]]},
    40  							{"metric":{"foo1":"bar1"},"values":[[1,"1"]]}
    41  						]`),
    42  			err: errors.New("expected 1 metrics but got 2"),
    43  		},
    44  		{
    45  			name: "same number of metrics but with different labels",
    46  			expected: json.RawMessage(`[
    47  							{"metric":{"foo":"bar"},"values":[[1,"1"]]}
    48  						]`),
    49  			actual: json.RawMessage(`[
    50  							{"metric":{"foo1":"bar1"},"values":[[1,"1"]]}
    51  						]`),
    52  			err: errors.New("expected metric {foo=\"bar\"} missing from actual response"),
    53  		},
    54  		{
    55  			name: "difference in number of samples",
    56  			expected: json.RawMessage(`[
    57  							{"metric":{"foo":"bar"},"values":[[1,"1"],[2,"2"]]}
    58  						]`),
    59  			actual: json.RawMessage(`[
    60  							{"metric":{"foo":"bar"},"values":[[1,"1"]]}
    61  						]`),
    62  			err: errors.New("expected 2 samples for metric {foo=\"bar\"} but got 1"),
    63  		},
    64  		{
    65  			name: "difference in sample timestamp",
    66  			expected: json.RawMessage(`[
    67  							{"metric":{"foo":"bar"},"values":[[1,"1"],[2,"2"]]}
    68  						]`),
    69  			actual: json.RawMessage(`[
    70  							{"metric":{"foo":"bar"},"values":[[1,"1"],[3,"2"]]}
    71  						]`),
    72  			// timestamps are parsed from seconds to ms which are then added to errors as is so adding 3 0s to expected error.
    73  			err: errors.New("sample pair not matching for metric {foo=\"bar\"}: expected timestamp 2 but got 3"),
    74  		},
    75  		{
    76  			name: "difference in sample value",
    77  			expected: json.RawMessage(`[
    78  							{"metric":{"foo":"bar"},"values":[[1,"1"],[2,"2"]]}
    79  						]`),
    80  			actual: json.RawMessage(`[
    81  							{"metric":{"foo":"bar"},"values":[[1,"1"],[2,"3"]]}
    82  						]`),
    83  			err: errors.New("sample pair not matching for metric {foo=\"bar\"}: expected value 2 for timestamp 2 but got 3"),
    84  		},
    85  		{
    86  			name: "correct samples",
    87  			expected: json.RawMessage(`[
    88  							{"metric":{"foo":"bar"},"values":[[1,"1"],[2,"2"]]}
    89  						]`),
    90  			actual: json.RawMessage(`[
    91  							{"metric":{"foo":"bar"},"values":[[1,"1"],[2,"2"]]}
    92  						]`),
    93  		},
    94  	} {
    95  		t.Run(tc.name, func(t *testing.T) {
    96  			err := compareMatrix(tc.expected, tc.actual, SampleComparisonOptions{})
    97  			if tc.err == nil {
    98  				require.NoError(t, err)
    99  				return
   100  			}
   101  			require.Error(t, err)
   102  			require.Equal(t, tc.err.Error(), err.Error())
   103  		})
   104  	}
   105  }
   106  
   107  func TestCompareVector(t *testing.T) {
   108  	for _, tc := range []struct {
   109  		name     string
   110  		expected json.RawMessage
   111  		actual   json.RawMessage
   112  		err      error
   113  	}{
   114  		{
   115  			name:     "no metrics",
   116  			expected: json.RawMessage(`[]`),
   117  			actual:   json.RawMessage(`[]`),
   118  		},
   119  		{
   120  			name: "no metrics in actual response",
   121  			expected: json.RawMessage(`[
   122  							{"metric":{"foo":"bar"},"value":[1,"1"]}
   123  						]`),
   124  			actual: json.RawMessage(`[]`),
   125  			err:    errors.New("expected 1 metrics but got 0"),
   126  		},
   127  		{
   128  			name: "extra metric in actual response",
   129  			expected: json.RawMessage(`[
   130  							{"metric":{"foo":"bar"},"value":[1,"1"]}
   131  						]`),
   132  			actual: json.RawMessage(`[
   133  							{"metric":{"foo":"bar"},"value":[1,"1"]},
   134  							{"metric":{"foo1":"bar1"},"value":[1,"1"]}
   135  						]`),
   136  			err: errors.New("expected 1 metrics but got 2"),
   137  		},
   138  		{
   139  			name: "same number of metrics but with different labels",
   140  			expected: json.RawMessage(`[
   141  							{"metric":{"foo":"bar"},"value":[1,"1"]}
   142  						]`),
   143  			actual: json.RawMessage(`[
   144  							{"metric":{"foo1":"bar1"},"value":[1,"1"]}
   145  						]`),
   146  			err: errors.New("expected metric {foo=\"bar\"} missing from actual response"),
   147  		},
   148  		{
   149  			name: "difference in sample timestamp",
   150  			expected: json.RawMessage(`[
   151  							{"metric":{"foo":"bar"},"value":[1,"1"]}
   152  						]`),
   153  			actual: json.RawMessage(`[
   154  							{"metric":{"foo":"bar"},"value":[2,"1"]}
   155  						]`),
   156  			err: errors.New("sample pair not matching for metric {foo=\"bar\"}: expected timestamp 1 but got 2"),
   157  		},
   158  		{
   159  			name: "difference in sample value",
   160  			expected: json.RawMessage(`[
   161  							{"metric":{"foo":"bar"},"value":[1,"1"]}
   162  						]`),
   163  			actual: json.RawMessage(`[
   164  							{"metric":{"foo":"bar"},"value":[1,"2"]}
   165  						]`),
   166  			err: errors.New("sample pair not matching for metric {foo=\"bar\"}: expected value 1 for timestamp 1 but got 2"),
   167  		},
   168  		{
   169  			name: "correct samples",
   170  			expected: json.RawMessage(`[
   171  							{"metric":{"foo":"bar"},"value":[1,"1"]}
   172  						]`),
   173  			actual: json.RawMessage(`[
   174  							{"metric":{"foo":"bar"},"value":[1,"1"]}
   175  						]`),
   176  		},
   177  	} {
   178  		t.Run(tc.name, func(t *testing.T) {
   179  			err := compareVector(tc.expected, tc.actual, SampleComparisonOptions{})
   180  			if tc.err == nil {
   181  				require.NoError(t, err)
   182  				return
   183  			}
   184  			require.Error(t, err)
   185  			require.Equal(t, tc.err.Error(), err.Error())
   186  		})
   187  	}
   188  }
   189  
   190  func TestCompareScalar(t *testing.T) {
   191  	for _, tc := range []struct {
   192  		name     string
   193  		expected json.RawMessage
   194  		actual   json.RawMessage
   195  		err      error
   196  	}{
   197  		{
   198  			name:     "difference in timestamp",
   199  			expected: json.RawMessage(`[1,"1"]`),
   200  			actual:   json.RawMessage(`[2,"1"]`),
   201  			err:      errors.New("expected timestamp 1 but got 2"),
   202  		},
   203  		{
   204  			name:     "difference in value",
   205  			expected: json.RawMessage(`[1,"1"]`),
   206  			actual:   json.RawMessage(`[1,"2"]`),
   207  			err:      errors.New("expected value 1 for timestamp 1 but got 2"),
   208  		},
   209  		{
   210  			name:     "correct values",
   211  			expected: json.RawMessage(`[1,"1"]`),
   212  			actual:   json.RawMessage(`[1,"1"]`),
   213  		},
   214  	} {
   215  		t.Run(tc.name, func(t *testing.T) {
   216  			err := compareScalar(tc.expected, tc.actual, SampleComparisonOptions{})
   217  			if tc.err == nil {
   218  				require.NoError(t, err)
   219  				return
   220  			}
   221  			require.Error(t, err)
   222  			require.Equal(t, tc.err.Error(), err.Error())
   223  		})
   224  	}
   225  }
   226  
   227  func TestCompareSamplesResponse(t *testing.T) {
   228  	now := model.Now().String()
   229  	for _, tc := range []struct {
   230  		name              string
   231  		tolerance         float64
   232  		expected          json.RawMessage
   233  		actual            json.RawMessage
   234  		err               error
   235  		useRelativeError  bool
   236  		skipRecentSamples time.Duration
   237  	}{
   238  		{
   239  			name: "difference in response status",
   240  			expected: json.RawMessage(`{
   241  							"status": "success",
   242  							"data": {"resultType":"scalar","result":[1,"1"]}
   243  						}`),
   244  			actual: json.RawMessage(`{
   245  							"status": "fail"
   246  						}`),
   247  			err: errors.New("expected status success but got fail"),
   248  		},
   249  		{
   250  			name: "difference in resultType",
   251  			expected: json.RawMessage(`{
   252  							"status": "success",
   253  							"data": {"resultType":"scalar","result":[1,"1"]}
   254  						}`),
   255  			actual: json.RawMessage(`{
   256  							"status": "success",
   257  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"1"]}]}
   258  						}`),
   259  			err: errors.New("expected resultType scalar but got vector"),
   260  		},
   261  		{
   262  			name: "unregistered resultType",
   263  			expected: json.RawMessage(`{
   264  							"status": "success",
   265  							"data": {"resultType":"new-scalar","result":[1,"1"]}
   266  						}`),
   267  			actual: json.RawMessage(`{
   268  							"status": "success",
   269  							"data": {"resultType":"new-scalar","result":[1,"1"]}
   270  						}`),
   271  			err: errors.New("resultType new-scalar not registered for comparison"),
   272  		},
   273  		{
   274  			name: "valid scalar response",
   275  			expected: json.RawMessage(`{
   276  							"status": "success",
   277  							"data": {"resultType":"scalar","result":[1,"1"]}
   278  						}`),
   279  			actual: json.RawMessage(`{
   280  							"status": "success",
   281  							"data": {"resultType":"scalar","result":[1,"1"]}
   282  						}`),
   283  		},
   284  		{
   285  			name:      "should pass if values are slightly different but within the tolerance",
   286  			tolerance: 0.000001,
   287  			expected: json.RawMessage(`{
   288  							"status": "success",
   289  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"773054.5916666666"]}]}
   290  						}`),
   291  			actual: json.RawMessage(`{
   292  							"status": "success",
   293  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"773054.59166667"]}]}
   294  						}`),
   295  		},
   296  		{
   297  			name:      "should correctly compare NaN values with tolerance is disabled",
   298  			tolerance: 0,
   299  			expected: json.RawMessage(`{
   300  							"status": "success",
   301  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"NaN"]}]}
   302  						}`),
   303  			actual: json.RawMessage(`{
   304  							"status": "success",
   305  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"NaN"]}]}
   306  						}`),
   307  		},
   308  		{
   309  			name:      "should correctly compare NaN values with tolerance is enabled",
   310  			tolerance: 0.000001,
   311  			expected: json.RawMessage(`{
   312  							"status": "success",
   313  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"NaN"]}]}
   314  						}`),
   315  			actual: json.RawMessage(`{
   316  							"status": "success",
   317  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"NaN"]}]}
   318  						}`),
   319  		},
   320  		{
   321  			name: "should correctly compare Inf values",
   322  			expected: json.RawMessage(`{
   323  							"status": "success",
   324  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"Inf"]}]}
   325  						}`),
   326  			actual: json.RawMessage(`{
   327  							"status": "success",
   328  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"Inf"]}]}
   329  						}`),
   330  		},
   331  		{
   332  			name: "should correctly compare -Inf values",
   333  			expected: json.RawMessage(`{
   334  							"status": "success",
   335  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"-Inf"]}]}
   336  						}`),
   337  			actual: json.RawMessage(`{
   338  							"status": "success",
   339  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"-Inf"]}]}
   340  						}`),
   341  		},
   342  		{
   343  			name: "should correctly compare +Inf values",
   344  			expected: json.RawMessage(`{
   345  							"status": "success",
   346  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"+Inf"]}]}
   347  						}`),
   348  			actual: json.RawMessage(`{
   349  							"status": "success",
   350  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"+Inf"]}]}
   351  						}`),
   352  		},
   353  		{
   354  			name:      "should fail if values are significantly different, over the tolerance",
   355  			tolerance: 0.000001,
   356  			expected: json.RawMessage(`{
   357  							"status": "success",
   358  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"773054.5916666666"]}]}
   359  						}`),
   360  			actual: json.RawMessage(`{
   361  							"status": "success",
   362  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"773054.789"]}]}
   363  						}`),
   364  			err: errors.New(`sample pair not matching for metric {foo="bar"}: expected value 773054.5916666666 for timestamp 1 but got 773054.789`),
   365  		},
   366  		{
   367  			name:      "should fail if large values are significantly different, over the tolerance without using relative error",
   368  			tolerance: 1e-14,
   369  			expected: json.RawMessage(`{
   370  							"status": "success",
   371  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"4.923488536785282e+41"]}]}
   372  						}`),
   373  			actual: json.RawMessage(`{
   374  							"status": "success",
   375  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"4.923488536785281e+41"]}]}
   376  						}`),
   377  			err: errors.New(`sample pair not matching for metric {foo="bar"}: expected value 492348853678528200000000000000000000000000 for timestamp 1 but got 492348853678528100000000000000000000000000`),
   378  		},
   379  		{
   380  			name:      "should not fail if large values are significantly different, over the tolerance using relative error",
   381  			tolerance: 1e-14,
   382  			expected: json.RawMessage(`{
   383  							"status": "success",
   384  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"4.923488536785282e+41"]}]}
   385  						}`),
   386  			actual: json.RawMessage(`{
   387  							"status": "success",
   388  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[1,"4.923488536785281e+41"]}]}
   389  						}`),
   390  			useRelativeError: true,
   391  		},
   392  		{
   393  			name: "should not fail when the sample is recent and configured to skip",
   394  			expected: json.RawMessage(`{
   395  							"status": "success",
   396  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[` + now + `,"10"]}]}
   397  						}`),
   398  			actual: json.RawMessage(`{
   399  							"status": "success",
   400  							"data": {"resultType":"vector","result":[{"metric":{"foo":"bar"},"value":[` + now + `,"5"]}]}
   401  						}`),
   402  			skipRecentSamples: time.Hour,
   403  		},
   404  	} {
   405  		t.Run(tc.name, func(t *testing.T) {
   406  			samplesComparator := NewSamplesComparator(SampleComparisonOptions{
   407  				Tolerance:         tc.tolerance,
   408  				UseRelativeError:  tc.useRelativeError,
   409  				SkipRecentSamples: tc.skipRecentSamples,
   410  			})
   411  			err := samplesComparator.Compare(tc.expected, tc.actual)
   412  			if tc.err == nil {
   413  				require.NoError(t, err)
   414  				return
   415  			}
   416  			require.Error(t, err)
   417  			require.Equal(t, tc.err.Error(), err.Error())
   418  		})
   419  	}
   420  }