github.com/go-graphite/carbonapi@v0.17.0/zipper/broadcast/broadcast_group_test.go (about)

     1  package broadcast
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"math"
     7  	"reflect"
     8  	"sort"
     9  	"testing"
    10  	"time"
    11  
    12  	"github.com/ansel1/merry"
    13  
    14  	"github.com/go-graphite/carbonapi/zipper/dummy"
    15  	"github.com/go-graphite/carbonapi/zipper/types"
    16  
    17  	protov3 "github.com/go-graphite/protocol/carbonapi_v3_pb"
    18  	"github.com/lomik/zapwriter"
    19  	"go.uber.org/zap"
    20  )
    21  
    22  var logger *zap.Logger
    23  var timeouts types.Timeouts
    24  
    25  func init() {
    26  	defaultLoggerConfig := zapwriter.Config{
    27  		Logger:           "",
    28  		File:             "stdout",
    29  		Level:            "debug",
    30  		Encoding:         "json",
    31  		EncodingTime:     "iso8601",
    32  		EncodingDuration: "seconds",
    33  	}
    34  
    35  	_ = zapwriter.ApplyConfig([]zapwriter.Config{defaultLoggerConfig})
    36  
    37  	logger = zapwriter.Logger("test")
    38  	timeouts = types.Timeouts{
    39  		Find:    1000 * time.Second,
    40  		Render:  1000 * time.Second,
    41  		Connect: 1000 * time.Second,
    42  	}
    43  }
    44  
    45  func errorsAreEqual(e1, e2 merry.Error) bool {
    46  	return merry.Is(e1, e2)
    47  }
    48  
    49  type testCaseNew struct {
    50  	name        string
    51  	servers     []types.BackendServer
    52  	expectedErr merry.Error
    53  }
    54  
    55  func TestNewBroadcastGroup(t *testing.T) {
    56  	tests := []testCaseNew{
    57  		{
    58  			name:        "no servers",
    59  			expectedErr: types.ErrNoServersSpecified,
    60  		},
    61  		{
    62  			name: "some servers",
    63  			servers: []types.BackendServer{
    64  				dummy.NewDummyClient("client1", []string{"backend1", "backend2"}, 0),
    65  			},
    66  		},
    67  	}
    68  
    69  	for _, tt := range tests {
    70  		t.Run(tt.name, func(t *testing.T) {
    71  			b, err := NewBroadcastGroup(logger, tt.name, true, tt.servers, 60, 500, 100, timeouts, false, false)
    72  			if !errorsAreEqual(err, tt.expectedErr) {
    73  				t.Fatalf("unexpected error %v, expected %v", err, tt.expectedErr)
    74  			}
    75  			_ = b
    76  		})
    77  	}
    78  }
    79  
    80  type testCaseProbe struct {
    81  	name            string
    82  	servers         []types.BackendServer
    83  	clientResponses map[string]dummy.ProbeResponse
    84  	response        []string
    85  	expectedErr     merry.Error
    86  }
    87  
    88  func TestProbeTLDs(t *testing.T) {
    89  	tests := []testCaseProbe{
    90  		{
    91  			name: "two backends different data",
    92  			servers: []types.BackendServer{
    93  				dummy.NewDummyClient("client1", []string{"backend1", "backend2"}, 1),
    94  				dummy.NewDummyClient("client2", []string{"backend3", "backend4"}, 1),
    95  			},
    96  			clientResponses: map[string]dummy.ProbeResponse{
    97  				"client1": {
    98  					Response: []string{"a", "b", "c"},
    99  				},
   100  				"client2": {
   101  					Response: []string{"a", "d", "e"},
   102  				},
   103  			},
   104  			response:    []string{"a", "b", "c", "d", "e"},
   105  			expectedErr: nil,
   106  		},
   107  	}
   108  
   109  	for _, tt := range tests {
   110  		b, err := NewBroadcastGroup(logger, tt.name, true, tt.servers, 60, 500, 100, timeouts, false, false)
   111  		if err != nil {
   112  			t.Fatalf("unexpected error %v", err)
   113  		}
   114  
   115  		for i := range tt.servers {
   116  			name := fmt.Sprintf("client%v", i+1)
   117  			s := tt.servers[i].(*dummy.DummyClient)
   118  			s.SetTLDResponse(tt.clientResponses[name])
   119  		}
   120  
   121  		ctx := context.Background()
   122  
   123  		t.Run(tt.name, func(t *testing.T) {
   124  			res, err := b.ProbeTLDs(ctx)
   125  			if !errorsAreEqual(err, tt.expectedErr) {
   126  				t.Fatalf("unexpected error %v, expected %v", err, tt.expectedErr)
   127  			}
   128  
   129  			if len(res) != len(tt.response) {
   130  				t.Fatalf("different amount of responses %v, expected %v", res, tt.response)
   131  			}
   132  
   133  			sort.Strings(res)
   134  			sort.Strings(tt.response)
   135  			for i := range res {
   136  				if res[i] != tt.response[i] {
   137  					t.Errorf("got %v, expected %v", res[i], tt.response[i])
   138  				}
   139  			}
   140  		})
   141  	}
   142  }
   143  
   144  type testCaseFetch struct {
   145  	name           string
   146  	servers        []types.BackendServer
   147  	fetchRequest   *protov3.MultiFetchRequest
   148  	fetchResponses map[string]dummy.FetchResponse
   149  
   150  	expectedErr      merry.Error
   151  	expectedResponse *protov3.MultiFetchResponse
   152  }
   153  
   154  func TestFetchRequests(t *testing.T) {
   155  	tests := []testCaseFetch{
   156  		{
   157  			name: "two backends different data",
   158  			servers: []types.BackendServer{
   159  				dummy.NewDummyClient("client1", []string{"backend1", "backend2"}, 1),
   160  				dummy.NewDummyClient("client2", []string{"backend3", "backend4"}, 1),
   161  			},
   162  			fetchRequest: &protov3.MultiFetchRequest{
   163  				Metrics: []protov3.FetchRequest{
   164  					{
   165  						Name:           "foo*",
   166  						StartTime:      0,
   167  						StopTime:       120,
   168  						PathExpression: "foo*",
   169  					},
   170  				},
   171  			},
   172  			fetchResponses: map[string]dummy.FetchResponse{
   173  				"client1": {
   174  					Response: &protov3.MultiFetchResponse{
   175  						Metrics: []protov3.FetchResponse{
   176  							{
   177  								Name:              "foo",
   178  								PathExpression:    "foo*",
   179  								ConsolidationFunc: "avg",
   180  								StartTime:         0,
   181  								StopTime:          120,
   182  								StepTime:          60,
   183  								XFilesFactor:      0.5,
   184  								Values:            []float64{0, 1, 2},
   185  							},
   186  						},
   187  					},
   188  					Stats:  &types.Stats{},
   189  					Errors: nil,
   190  				},
   191  				"client2": {
   192  					Response: &protov3.MultiFetchResponse{
   193  						Metrics: []protov3.FetchResponse{
   194  							{
   195  								Name:              "foo2",
   196  								PathExpression:    "foo*",
   197  								ConsolidationFunc: "avg",
   198  								StartTime:         0,
   199  								StopTime:          120,
   200  								StepTime:          60,
   201  								XFilesFactor:      0.5,
   202  								Values:            []float64{0, 1, 2},
   203  							},
   204  						},
   205  					},
   206  					Stats:  &types.Stats{},
   207  					Errors: nil,
   208  				},
   209  			},
   210  
   211  			expectedResponse: &protov3.MultiFetchResponse{
   212  				Metrics: []protov3.FetchResponse{
   213  					{
   214  						Name:              "foo",
   215  						PathExpression:    "foo*",
   216  						ConsolidationFunc: "avg",
   217  						StartTime:         0,
   218  						StopTime:          180,
   219  						StepTime:          60,
   220  						XFilesFactor:      0.5,
   221  						Values:            []float64{0, 1, 2},
   222  					},
   223  					{
   224  						Name:              "foo2",
   225  						PathExpression:    "foo*",
   226  						ConsolidationFunc: "avg",
   227  						StartTime:         0,
   228  						StopTime:          180,
   229  						StepTime:          60,
   230  						XFilesFactor:      0.5,
   231  						Values:            []float64{0, 1, 2},
   232  					},
   233  				},
   234  			},
   235  		},
   236  		{
   237  			name: "two backends same data",
   238  			servers: []types.BackendServer{
   239  				dummy.NewDummyClient("client1", []string{"backend1", "backend2"}, 1),
   240  				dummy.NewDummyClient("client2", []string{"backend3", "backend4"}, 1),
   241  			},
   242  			fetchRequest: &protov3.MultiFetchRequest{
   243  				Metrics: []protov3.FetchRequest{
   244  					{
   245  						Name:           "foo",
   246  						StartTime:      0,
   247  						StopTime:       120,
   248  						PathExpression: "foo",
   249  					},
   250  				},
   251  			},
   252  			fetchResponses: map[string]dummy.FetchResponse{
   253  				"client1": {
   254  					Response: &protov3.MultiFetchResponse{
   255  						Metrics: []protov3.FetchResponse{
   256  							{
   257  								Name:              "foo",
   258  								PathExpression:    "foo",
   259  								ConsolidationFunc: "avg",
   260  								StartTime:         0,
   261  								StopTime:          120,
   262  								StepTime:          60,
   263  								XFilesFactor:      0.5,
   264  								Values:            []float64{0, 1, 2},
   265  							},
   266  						},
   267  					},
   268  					Stats:  &types.Stats{},
   269  					Errors: nil,
   270  				},
   271  				"client2": {
   272  					Response: &protov3.MultiFetchResponse{
   273  						Metrics: []protov3.FetchResponse{
   274  							{
   275  								Name:              "foo",
   276  								PathExpression:    "foo",
   277  								ConsolidationFunc: "avg",
   278  								StartTime:         0,
   279  								StopTime:          120,
   280  								StepTime:          60,
   281  								XFilesFactor:      0.5,
   282  								Values:            []float64{0, 1, 2},
   283  							},
   284  						},
   285  					},
   286  					Stats:  &types.Stats{},
   287  					Errors: nil,
   288  				},
   289  			},
   290  			expectedResponse: &protov3.MultiFetchResponse{
   291  				Metrics: []protov3.FetchResponse{
   292  					{
   293  						Name:              "foo",
   294  						PathExpression:    "foo",
   295  						ConsolidationFunc: "avg",
   296  						StartTime:         0,
   297  						StopTime:          180,
   298  						StepTime:          60,
   299  						XFilesFactor:      0.5,
   300  						Values:            []float64{0, 1, 2},
   301  					},
   302  				},
   303  			},
   304  		},
   305  		{
   306  			name: "two backends merge data",
   307  			servers: []types.BackendServer{
   308  				dummy.NewDummyClient("client1", []string{"backend1", "backend2"}, 1),
   309  				dummy.NewDummyClient("client2", []string{"backend3", "backend4"}, 1),
   310  			},
   311  			fetchRequest: &protov3.MultiFetchRequest{
   312  				Metrics: []protov3.FetchRequest{
   313  					{
   314  						Name:           "foo",
   315  						StartTime:      0,
   316  						StopTime:       120,
   317  						PathExpression: "foo",
   318  					},
   319  				},
   320  			},
   321  			fetchResponses: map[string]dummy.FetchResponse{
   322  				"client1": {
   323  					Response: &protov3.MultiFetchResponse{
   324  						Metrics: []protov3.FetchResponse{
   325  							{
   326  								Name:              "foo",
   327  								PathExpression:    "foo",
   328  								ConsolidationFunc: "avg",
   329  								StartTime:         0,
   330  								StopTime:          120,
   331  								StepTime:          60,
   332  								XFilesFactor:      0.5,
   333  								Values:            []float64{0, math.NaN(), 2},
   334  							},
   335  						},
   336  					},
   337  					Stats:  &types.Stats{},
   338  					Errors: nil,
   339  				},
   340  				"client2": {
   341  					Response: &protov3.MultiFetchResponse{
   342  						Metrics: []protov3.FetchResponse{
   343  							{
   344  								Name:              "foo",
   345  								PathExpression:    "foo",
   346  								ConsolidationFunc: "avg",
   347  								StartTime:         0,
   348  								StopTime:          120,
   349  								StepTime:          60,
   350  								XFilesFactor:      0.5,
   351  								Values:            []float64{0, 1, math.NaN()},
   352  							},
   353  						},
   354  					},
   355  					Stats:  &types.Stats{},
   356  					Errors: nil,
   357  				},
   358  			},
   359  			expectedResponse: &protov3.MultiFetchResponse{
   360  				Metrics: []protov3.FetchResponse{
   361  					{
   362  						Name:              "foo",
   363  						PathExpression:    "foo",
   364  						ConsolidationFunc: "avg",
   365  						StartTime:         0,
   366  						StopTime:          180,
   367  						StepTime:          60,
   368  						XFilesFactor:      0.5,
   369  						Values:            []float64{0, 1, 2},
   370  					},
   371  				},
   372  			},
   373  		},
   374  		{
   375  			name: "two backends different length data",
   376  			servers: []types.BackendServer{
   377  				dummy.NewDummyClient("client1", []string{"backend1", "backend2"}, 1),
   378  				dummy.NewDummyClient("client2", []string{"backend3", "backend4"}, 1),
   379  			},
   380  			fetchRequest: &protov3.MultiFetchRequest{
   381  				Metrics: []protov3.FetchRequest{
   382  					{
   383  						Name:           "foo",
   384  						StartTime:      0,
   385  						StopTime:       180,
   386  						PathExpression: "foo",
   387  					},
   388  				},
   389  			},
   390  			fetchResponses: map[string]dummy.FetchResponse{
   391  				"client1": {
   392  					Response: &protov3.MultiFetchResponse{
   393  						Metrics: []protov3.FetchResponse{
   394  							{
   395  								Name:              "foo",
   396  								PathExpression:    "foo",
   397  								ConsolidationFunc: "avg",
   398  								StartTime:         0,
   399  								StopTime:          180,
   400  								StepTime:          60,
   401  								XFilesFactor:      0.5,
   402  								Values:            []float64{0, 1, 2, 3},
   403  							},
   404  						},
   405  					},
   406  					Stats:  &types.Stats{},
   407  					Errors: nil,
   408  				},
   409  				"client2": {
   410  					Response: &protov3.MultiFetchResponse{
   411  						Metrics: []protov3.FetchResponse{
   412  							{
   413  								Name:              "foo",
   414  								PathExpression:    "foo",
   415  								ConsolidationFunc: "avg",
   416  								StartTime:         0,
   417  								StopTime:          120,
   418  								StepTime:          60,
   419  								XFilesFactor:      0.5,
   420  								Values:            []float64{0, 1, 2},
   421  							},
   422  						},
   423  					},
   424  					Stats:  &types.Stats{},
   425  					Errors: nil,
   426  				},
   427  			},
   428  			expectedResponse: &protov3.MultiFetchResponse{
   429  				Metrics: []protov3.FetchResponse{
   430  					{
   431  						Name:              "foo",
   432  						PathExpression:    "foo",
   433  						ConsolidationFunc: "avg",
   434  						StartTime:         0,
   435  						StopTime:          240,
   436  						StepTime:          60,
   437  						XFilesFactor:      0.5,
   438  						Values:            []float64{0, 1, 2, 3},
   439  					},
   440  				},
   441  			},
   442  		},
   443  		{
   444  			name: "many backends, different data",
   445  			servers: []types.BackendServer{
   446  				dummy.NewDummyClient("client1", []string{"backend1", "backend2"}, 1),
   447  				dummy.NewDummyClient("client2", []string{"backend3", "backend4"}, 1),
   448  				dummy.NewDummyClient("client3", []string{"backend5", "backend6"}, 1),
   449  				dummy.NewDummyClient("client4", []string{"backend7", "backend8"}, 1),
   450  				dummy.NewDummyClient("client5", []string{"backend9", "backend10"}, 1),
   451  				dummy.NewDummyClient("client6", []string{"backend11", "backend12"}, 1),
   452  				dummy.NewDummyClient("client7", []string{"backend13", "backend14"}, 1),
   453  				dummy.NewDummyClient("client8", []string{"backend15", "backend16"}, 1),
   454  				dummy.NewDummyClient("client9", []string{"backend17", "backend18"}, 1),
   455  				dummy.NewDummyClient("client10", []string{"backend19", "backend20"}, 1),
   456  				dummy.NewDummyClient("client11", []string{"backend21", "backend22"}, 1),
   457  				dummy.NewDummyClient("client12", []string{"backend23", "backend24"}, 1),
   458  				dummy.NewDummyClient("client13", []string{"backend25", "backend26"}, 1),
   459  				dummy.NewDummyClient("client14", []string{"backend27", "backend28"}, 1),
   460  				dummy.NewDummyClient("client15", []string{"backend29", "backend30"}, 1),
   461  				dummy.NewDummyClient("client16", []string{"backend31", "backend32"}, 1),
   462  				dummy.NewDummyClient("client17", []string{"backend33", "backend34"}, 1),
   463  				dummy.NewDummyClient("client18", []string{"backend35", "backend36"}, 1),
   464  				dummy.NewDummyClient("client19", []string{"backend37", "backend38"}, 1),
   465  				dummy.NewDummyClient("client20", []string{"backend39", "backend40"}, 1),
   466  				dummy.NewDummyClient("client21", []string{"backend41", "backend42"}, 1),
   467  				dummy.NewDummyClient("client22", []string{"backend43", "backend44"}, 1),
   468  			},
   469  			fetchRequest: &protov3.MultiFetchRequest{
   470  				Metrics: []protov3.FetchRequest{
   471  					{
   472  						Name:           "foo*",
   473  						StartTime:      0,
   474  						StopTime:       180,
   475  						PathExpression: "foo*",
   476  					},
   477  				},
   478  			},
   479  			fetchResponses: map[string]dummy.FetchResponse{
   480  				"client1": {
   481  					Response: &protov3.MultiFetchResponse{
   482  						Metrics: []protov3.FetchResponse{
   483  							{
   484  								Name:              "foo",
   485  								PathExpression:    "foo*",
   486  								ConsolidationFunc: "avg",
   487  								StartTime:         0,
   488  								StopTime:          180,
   489  								StepTime:          60,
   490  								XFilesFactor:      0.5,
   491  								Values:            []float64{0, 1, 2, 3},
   492  							},
   493  						},
   494  					},
   495  					Stats:  &types.Stats{},
   496  					Errors: nil,
   497  				},
   498  				"client2": {
   499  					Response: &protov3.MultiFetchResponse{
   500  						Metrics: []protov3.FetchResponse{
   501  							{
   502  								Name:              "foo",
   503  								PathExpression:    "foo*",
   504  								ConsolidationFunc: "avg",
   505  								StartTime:         0,
   506  								StopTime:          120,
   507  								StepTime:          60,
   508  								XFilesFactor:      0.5,
   509  								Values:            []float64{0, math.NaN(), 2},
   510  							},
   511  							{
   512  								Name:              "foo2",
   513  								PathExpression:    "foo*",
   514  								ConsolidationFunc: "avg",
   515  								StartTime:         0,
   516  								StopTime:          180,
   517  								StepTime:          60,
   518  								XFilesFactor:      0.5,
   519  								Values:            []float64{0, 1, 2, math.NaN()},
   520  							},
   521  						},
   522  					},
   523  					Stats:  &types.Stats{},
   524  					Errors: nil,
   525  				},
   526  				"client3": {
   527  					Response: &protov3.MultiFetchResponse{
   528  						Metrics: []protov3.FetchResponse{
   529  							{
   530  								Name:              "foo",
   531  								PathExpression:    "foo*",
   532  								ConsolidationFunc: "avg",
   533  								StartTime:         0,
   534  								StopTime:          60,
   535  								StepTime:          60,
   536  								XFilesFactor:      0.5,
   537  								Values:            []float64{0, 1},
   538  							},
   539  							{
   540  								Name:              "foo2",
   541  								PathExpression:    "foo*",
   542  								ConsolidationFunc: "avg",
   543  								StartTime:         0,
   544  								StopTime:          180,
   545  								StepTime:          60,
   546  								XFilesFactor:      0.5,
   547  								Values:            []float64{0, 1, 2, 3},
   548  							},
   549  						},
   550  					},
   551  					Stats:  &types.Stats{},
   552  					Errors: nil,
   553  				},
   554  				"client4": {
   555  					Response: &protov3.MultiFetchResponse{
   556  						Metrics: []protov3.FetchResponse{
   557  							{
   558  								Name:              "foo",
   559  								PathExpression:    "foo*",
   560  								ConsolidationFunc: "avg",
   561  								StartTime:         0,
   562  								StopTime:          120,
   563  								StepTime:          60,
   564  								XFilesFactor:      0.5,
   565  								Values:            []float64{0, 1, 2},
   566  							},
   567  							{
   568  								Name:              "foo2",
   569  								PathExpression:    "foo*",
   570  								ConsolidationFunc: "avg",
   571  								StartTime:         0,
   572  								StopTime:          180,
   573  								StepTime:          60,
   574  								XFilesFactor:      0.5,
   575  								Values:            []float64{0, 1, 2, 3},
   576  							},
   577  						},
   578  					},
   579  					Stats:  &types.Stats{},
   580  					Errors: nil,
   581  				},
   582  				"client5": {
   583  					Response: &protov3.MultiFetchResponse{
   584  						Metrics: []protov3.FetchResponse{
   585  							{
   586  								Name:              "foo",
   587  								PathExpression:    "foo*",
   588  								ConsolidationFunc: "avg",
   589  								StartTime:         0,
   590  								StopTime:          120,
   591  								StepTime:          60,
   592  								XFilesFactor:      0.5,
   593  								Values:            []float64{0, 1, 2},
   594  							},
   595  							{
   596  								Name:              "foo2",
   597  								PathExpression:    "foo*",
   598  								ConsolidationFunc: "avg",
   599  								StartTime:         0,
   600  								StopTime:          180,
   601  								StepTime:          60,
   602  								XFilesFactor:      0.5,
   603  								Values:            []float64{0, 1, 2, 3},
   604  							},
   605  						},
   606  					},
   607  					Stats:  &types.Stats{},
   608  					Errors: nil,
   609  				},
   610  				"client6": {
   611  					Response: &protov3.MultiFetchResponse{
   612  						Metrics: []protov3.FetchResponse{
   613  							{
   614  								Name:              "foo",
   615  								PathExpression:    "foo*",
   616  								ConsolidationFunc: "avg",
   617  								StartTime:         0,
   618  								StopTime:          120,
   619  								StepTime:          60,
   620  								XFilesFactor:      0.5,
   621  								Values:            []float64{0, 1, 2},
   622  							},
   623  							{
   624  								Name:              "foo2",
   625  								PathExpression:    "foo*",
   626  								ConsolidationFunc: "avg",
   627  								StartTime:         0,
   628  								StopTime:          180,
   629  								StepTime:          60,
   630  								XFilesFactor:      0.5,
   631  								Values:            []float64{0, 1, 2, 3},
   632  							},
   633  						},
   634  					},
   635  					Stats:  &types.Stats{},
   636  					Errors: nil,
   637  				},
   638  				"client7": {
   639  					Response: &protov3.MultiFetchResponse{
   640  						Metrics: []protov3.FetchResponse{
   641  							{
   642  								Name:              "foo",
   643  								PathExpression:    "foo*",
   644  								ConsolidationFunc: "avg",
   645  								StartTime:         0,
   646  								StopTime:          120,
   647  								StepTime:          60,
   648  								XFilesFactor:      0.5,
   649  								Values:            []float64{0, 1, 2},
   650  							},
   651  							{
   652  								Name:              "foo2",
   653  								PathExpression:    "foo*",
   654  								ConsolidationFunc: "avg",
   655  								StartTime:         0,
   656  								StopTime:          180,
   657  								StepTime:          60,
   658  								XFilesFactor:      0.5,
   659  								Values:            []float64{0, 1, 2, 3},
   660  							},
   661  						},
   662  					},
   663  					Stats:  &types.Stats{},
   664  					Errors: nil,
   665  				},
   666  				"client8": {
   667  					Response: &protov3.MultiFetchResponse{
   668  						Metrics: []protov3.FetchResponse{
   669  							{
   670  								Name:              "foo",
   671  								PathExpression:    "foo*",
   672  								ConsolidationFunc: "avg",
   673  								StartTime:         0,
   674  								StopTime:          120,
   675  								StepTime:          60,
   676  								XFilesFactor:      0.5,
   677  								Values:            []float64{0, 1, 2},
   678  							},
   679  							{
   680  								Name:              "foo2",
   681  								PathExpression:    "foo*",
   682  								ConsolidationFunc: "avg",
   683  								StartTime:         0,
   684  								StopTime:          180,
   685  								StepTime:          60,
   686  								XFilesFactor:      0.5,
   687  								Values:            []float64{0, 1, 2, 3},
   688  							},
   689  						},
   690  					},
   691  					Stats:  &types.Stats{},
   692  					Errors: nil,
   693  				},
   694  				"client9": {
   695  					Response: &protov3.MultiFetchResponse{
   696  						Metrics: []protov3.FetchResponse{
   697  							{
   698  								Name:              "foo",
   699  								PathExpression:    "foo*",
   700  								ConsolidationFunc: "avg",
   701  								StartTime:         0,
   702  								StopTime:          120,
   703  								StepTime:          60,
   704  								XFilesFactor:      0.5,
   705  								Values:            []float64{0, 1, 2},
   706  							},
   707  							{
   708  								Name:              "foo2",
   709  								PathExpression:    "foo*",
   710  								ConsolidationFunc: "avg",
   711  								StartTime:         0,
   712  								StopTime:          180,
   713  								StepTime:          60,
   714  								XFilesFactor:      0.5,
   715  								Values:            []float64{0, 1, 2, 3},
   716  							},
   717  						},
   718  					},
   719  					Stats:  &types.Stats{},
   720  					Errors: nil,
   721  				},
   722  				"client10": {
   723  					Response: &protov3.MultiFetchResponse{
   724  						Metrics: []protov3.FetchResponse{
   725  							{
   726  								Name:              "foo",
   727  								PathExpression:    "foo*",
   728  								ConsolidationFunc: "avg",
   729  								StartTime:         0,
   730  								StopTime:          120,
   731  								StepTime:          60,
   732  								XFilesFactor:      0.5,
   733  								Values:            []float64{0, 1, 2},
   734  							},
   735  							{
   736  								Name:              "foo2",
   737  								PathExpression:    "foo*",
   738  								ConsolidationFunc: "avg",
   739  								StartTime:         0,
   740  								StopTime:          180,
   741  								StepTime:          60,
   742  								XFilesFactor:      0.5,
   743  								Values:            []float64{0, 1, 2, 3},
   744  							},
   745  						},
   746  					},
   747  					Stats:  &types.Stats{},
   748  					Errors: nil,
   749  				},
   750  				"client11": {
   751  					Response: &protov3.MultiFetchResponse{
   752  						Metrics: []protov3.FetchResponse{
   753  							{
   754  								Name:              "foo",
   755  								PathExpression:    "foo*",
   756  								ConsolidationFunc: "avg",
   757  								StartTime:         0,
   758  								StopTime:          120,
   759  								StepTime:          60,
   760  								XFilesFactor:      0.5,
   761  								Values:            []float64{0, 1, 2},
   762  							},
   763  							{
   764  								Name:              "foo2",
   765  								PathExpression:    "foo*",
   766  								ConsolidationFunc: "avg",
   767  								StartTime:         0,
   768  								StopTime:          180,
   769  								StepTime:          60,
   770  								XFilesFactor:      0.5,
   771  								Values:            []float64{0, 1, 2, 3},
   772  							},
   773  						},
   774  					},
   775  					Stats:  &types.Stats{},
   776  					Errors: nil,
   777  				},
   778  				"client12": {
   779  					Response: &protov3.MultiFetchResponse{
   780  						Metrics: []protov3.FetchResponse{
   781  							{
   782  								Name:              "foo",
   783  								PathExpression:    "foo*",
   784  								ConsolidationFunc: "avg",
   785  								StartTime:         0,
   786  								StopTime:          120,
   787  								StepTime:          60,
   788  								XFilesFactor:      0.5,
   789  								Values:            []float64{0, 1, 2},
   790  							},
   791  							{
   792  								Name:              "foo2",
   793  								PathExpression:    "foo*",
   794  								ConsolidationFunc: "avg",
   795  								StartTime:         0,
   796  								StopTime:          60,
   797  								StepTime:          60,
   798  								XFilesFactor:      0.5,
   799  								Values:            []float64{0},
   800  							},
   801  						},
   802  					},
   803  					Stats:  &types.Stats{},
   804  					Errors: nil,
   805  				},
   806  				"client13": {
   807  					Response: &protov3.MultiFetchResponse{
   808  						Metrics: []protov3.FetchResponse{
   809  							{
   810  								Name:              "foo",
   811  								PathExpression:    "foo*",
   812  								ConsolidationFunc: "avg",
   813  								StartTime:         0,
   814  								StopTime:          120,
   815  								StepTime:          60,
   816  								XFilesFactor:      0.5,
   817  								Values:            []float64{0, 1, 2},
   818  							},
   819  							{
   820  								Name:              "foo2",
   821  								PathExpression:    "foo*",
   822  								ConsolidationFunc: "avg",
   823  								StartTime:         0,
   824  								StopTime:          180,
   825  								StepTime:          60,
   826  								XFilesFactor:      0.5,
   827  								Values:            []float64{0, 1, 2, 3},
   828  							},
   829  						},
   830  					},
   831  					Stats:  &types.Stats{},
   832  					Errors: nil,
   833  				},
   834  				"client14": {
   835  					Response: &protov3.MultiFetchResponse{
   836  						Metrics: []protov3.FetchResponse{
   837  							{
   838  								Name:              "foo",
   839  								PathExpression:    "foo*",
   840  								ConsolidationFunc: "avg",
   841  								StartTime:         0,
   842  								StopTime:          120,
   843  								StepTime:          60,
   844  								XFilesFactor:      0.5,
   845  								Values:            []float64{0, math.NaN(), 2},
   846  							},
   847  							{
   848  								Name:              "foo2",
   849  								PathExpression:    "foo*",
   850  								ConsolidationFunc: "avg",
   851  								StartTime:         0,
   852  								StopTime:          180,
   853  								StepTime:          60,
   854  								XFilesFactor:      0.5,
   855  								Values:            []float64{0, 1, 2, 3},
   856  							},
   857  						},
   858  					},
   859  					Stats:  &types.Stats{},
   860  					Errors: nil,
   861  				},
   862  				"client15": {
   863  					Response: &protov3.MultiFetchResponse{
   864  						Metrics: []protov3.FetchResponse{
   865  							{
   866  								Name:              "foo",
   867  								PathExpression:    "foo*",
   868  								ConsolidationFunc: "avg",
   869  								StartTime:         0,
   870  								StopTime:          120,
   871  								StepTime:          60,
   872  								XFilesFactor:      0.5,
   873  								Values:            []float64{0, 1, 2},
   874  							},
   875  							{
   876  								Name:              "foo2",
   877  								PathExpression:    "foo*",
   878  								ConsolidationFunc: "avg",
   879  								StartTime:         0,
   880  								StopTime:          180,
   881  								StepTime:          60,
   882  								XFilesFactor:      0.5,
   883  								Values:            []float64{math.NaN(), 1, 2, 3},
   884  							},
   885  						},
   886  					},
   887  					Stats:  &types.Stats{},
   888  					Errors: nil,
   889  				},
   890  				"client16": {
   891  					Response: &protov3.MultiFetchResponse{
   892  						Metrics: []protov3.FetchResponse{
   893  							{
   894  								Name:              "foo",
   895  								PathExpression:    "foo*",
   896  								ConsolidationFunc: "avg",
   897  								StartTime:         0,
   898  								StopTime:          120,
   899  								StepTime:          60,
   900  								XFilesFactor:      0.5,
   901  								Values:            []float64{0, 1, 2},
   902  							},
   903  							{
   904  								Name:              "foo2",
   905  								PathExpression:    "foo*",
   906  								ConsolidationFunc: "avg",
   907  								StartTime:         0,
   908  								StopTime:          180,
   909  								StepTime:          60,
   910  								XFilesFactor:      0.5,
   911  								Values:            []float64{0, 1, 2, 3},
   912  							},
   913  						},
   914  					},
   915  					Stats:  &types.Stats{},
   916  					Errors: nil,
   917  				},
   918  				"client17": {
   919  					Response: &protov3.MultiFetchResponse{
   920  						Metrics: []protov3.FetchResponse{
   921  							{
   922  								Name:              "foo",
   923  								PathExpression:    "foo*",
   924  								ConsolidationFunc: "avg",
   925  								StartTime:         0,
   926  								StopTime:          120,
   927  								StepTime:          60,
   928  								XFilesFactor:      0.5,
   929  								Values:            []float64{0, 1, 2},
   930  							},
   931  							{
   932  								Name:              "foo2",
   933  								PathExpression:    "foo*",
   934  								ConsolidationFunc: "avg",
   935  								StartTime:         0,
   936  								StopTime:          180,
   937  								StepTime:          60,
   938  								XFilesFactor:      0.5,
   939  								Values:            []float64{0, 1, 2, 3},
   940  							},
   941  						},
   942  					},
   943  					Stats:  &types.Stats{},
   944  					Errors: nil,
   945  				},
   946  				"client18": {
   947  					Response: &protov3.MultiFetchResponse{
   948  						Metrics: []protov3.FetchResponse{
   949  							{
   950  								Name:              "foo",
   951  								PathExpression:    "foo*",
   952  								ConsolidationFunc: "avg",
   953  								StartTime:         0,
   954  								StopTime:          120,
   955  								StepTime:          60,
   956  								XFilesFactor:      0.5,
   957  								Values:            []float64{0, 1, 2},
   958  							},
   959  							{
   960  								Name:              "foo2",
   961  								PathExpression:    "foo*",
   962  								ConsolidationFunc: "avg",
   963  								StartTime:         0,
   964  								StopTime:          180,
   965  								StepTime:          60,
   966  								XFilesFactor:      0.5,
   967  								Values:            []float64{0, 1, 2, 3},
   968  							},
   969  						},
   970  					},
   971  					Stats:  &types.Stats{},
   972  					Errors: nil,
   973  				},
   974  				"client19": {
   975  					Response: &protov3.MultiFetchResponse{
   976  						Metrics: []protov3.FetchResponse{
   977  							{
   978  								Name:              "foo",
   979  								PathExpression:    "foo*",
   980  								ConsolidationFunc: "avg",
   981  								StartTime:         0,
   982  								StopTime:          120,
   983  								StepTime:          60,
   984  								XFilesFactor:      0.5,
   985  								Values:            []float64{0, 1, 2},
   986  							},
   987  							{
   988  								Name:              "foo2",
   989  								PathExpression:    "foo*",
   990  								ConsolidationFunc: "avg",
   991  								StartTime:         0,
   992  								StopTime:          180,
   993  								StepTime:          60,
   994  								XFilesFactor:      0.5,
   995  								Values:            []float64{0, 1, 2, 3},
   996  							},
   997  						},
   998  					},
   999  					Stats:  &types.Stats{},
  1000  					Errors: nil,
  1001  				},
  1002  				"client20": {
  1003  					Response: &protov3.MultiFetchResponse{
  1004  						Metrics: []protov3.FetchResponse{
  1005  							{
  1006  								Name:              "foo",
  1007  								PathExpression:    "foo*",
  1008  								ConsolidationFunc: "avg",
  1009  								StartTime:         0,
  1010  								StopTime:          120,
  1011  								StepTime:          60,
  1012  								XFilesFactor:      0.5,
  1013  								Values:            []float64{0, 1, 2},
  1014  							},
  1015  							{
  1016  								Name:              "foo2",
  1017  								PathExpression:    "foo*",
  1018  								ConsolidationFunc: "avg",
  1019  								StartTime:         0,
  1020  								StopTime:          180,
  1021  								StepTime:          60,
  1022  								XFilesFactor:      0.5,
  1023  								Values:            []float64{0, 1, 2, 3},
  1024  							},
  1025  						},
  1026  					},
  1027  					Stats:  &types.Stats{},
  1028  					Errors: nil,
  1029  				},
  1030  				"client21": {
  1031  					Response: &protov3.MultiFetchResponse{
  1032  						Metrics: []protov3.FetchResponse{
  1033  							{
  1034  								Name:              "foo",
  1035  								PathExpression:    "foo*",
  1036  								ConsolidationFunc: "avg",
  1037  								StartTime:         0,
  1038  								StopTime:          120,
  1039  								StepTime:          60,
  1040  								XFilesFactor:      0.5,
  1041  								Values:            []float64{0, 1, 2},
  1042  							},
  1043  							{
  1044  								Name:              "foo2",
  1045  								PathExpression:    "foo*",
  1046  								ConsolidationFunc: "avg",
  1047  								StartTime:         0,
  1048  								StopTime:          180,
  1049  								StepTime:          60,
  1050  								XFilesFactor:      0.5,
  1051  								Values:            []float64{0, 1, 2, 3},
  1052  							},
  1053  						},
  1054  					},
  1055  					Stats:  &types.Stats{},
  1056  					Errors: nil,
  1057  				},
  1058  				"client22": {
  1059  					Response: &protov3.MultiFetchResponse{
  1060  						Metrics: []protov3.FetchResponse{
  1061  							{
  1062  								Name:              "foo",
  1063  								PathExpression:    "foo*",
  1064  								ConsolidationFunc: "avg",
  1065  								StartTime:         0,
  1066  								StopTime:          120,
  1067  								StepTime:          60,
  1068  								XFilesFactor:      0.5,
  1069  								Values:            []float64{0, 1, 2},
  1070  							},
  1071  							{
  1072  								Name:              "foo2",
  1073  								PathExpression:    "foo*",
  1074  								ConsolidationFunc: "avg",
  1075  								StartTime:         0,
  1076  								StopTime:          180,
  1077  								StepTime:          60,
  1078  								XFilesFactor:      0.5,
  1079  								Values:            []float64{0, 1, 2, 3},
  1080  							},
  1081  						},
  1082  					},
  1083  					Stats:  &types.Stats{},
  1084  					Errors: nil,
  1085  				},
  1086  			},
  1087  			expectedResponse: &protov3.MultiFetchResponse{
  1088  				Metrics: []protov3.FetchResponse{
  1089  					{
  1090  						Name:              "foo",
  1091  						PathExpression:    "foo*",
  1092  						ConsolidationFunc: "avg",
  1093  						StartTime:         0,
  1094  						StopTime:          240,
  1095  						StepTime:          60,
  1096  						XFilesFactor:      0.5,
  1097  						Values:            []float64{0, 1, 2, 3},
  1098  					},
  1099  					{
  1100  						Name:              "foo2",
  1101  						PathExpression:    "foo*",
  1102  						ConsolidationFunc: "avg",
  1103  						StartTime:         0,
  1104  						StopTime:          240,
  1105  						StepTime:          60,
  1106  						XFilesFactor:      0.5,
  1107  						Values:            []float64{0, 1, 2, 3},
  1108  					},
  1109  				},
  1110  			},
  1111  		},
  1112  	}
  1113  
  1114  	for _, tt := range tests {
  1115  		b, err := New(
  1116  			WithLogger(logger),
  1117  			WithGroupName(tt.name),
  1118  			WithSplitMultipleRequests(false),
  1119  			WithBackends(tt.servers),
  1120  			WithPathCache(60),
  1121  			WithLimiter(500),
  1122  			WithMaxMetricsPerRequest(100),
  1123  			WithTimeouts(timeouts),
  1124  			WithTLDCache(true),
  1125  		)
  1126  		if err != nil {
  1127  			t.Fatalf("unepxected error %v", err)
  1128  		}
  1129  
  1130  		for i := range tt.servers {
  1131  			name := fmt.Sprintf("client%v", i+1)
  1132  			s := tt.servers[i].(*dummy.DummyClient)
  1133  			resp, ok := tt.fetchResponses[name]
  1134  			if ok {
  1135  				s.AddFetchResponse(tt.fetchRequest, resp.Response, resp.Stats, resp.Errors)
  1136  			}
  1137  		}
  1138  
  1139  		ctx := context.Background()
  1140  
  1141  		t.Run(tt.name, func(t *testing.T) {
  1142  			res, _, err := b.Fetch(ctx, tt.fetchRequest)
  1143  			if tt.expectedErr == nil {
  1144  				if err != nil {
  1145  					t.Errorf("unexpected error '%+v', expected %v", merry.Details(err), tt.expectedErr)
  1146  				}
  1147  			} else {
  1148  				if !errorsAreEqual(err, tt.expectedErr) {
  1149  					t.Errorf("unexpected error %v, expected %v", merry.Details(err), tt.expectedErr)
  1150  				}
  1151  			}
  1152  
  1153  			if res == nil {
  1154  				t.Fatal("result is nil")
  1155  			}
  1156  
  1157  			if len(res.Metrics) != len(tt.expectedResponse.Metrics) {
  1158  				t.Fatalf("different amount of responses %v, expected %v", res, tt.expectedResponse)
  1159  			}
  1160  
  1161  			sort.Slice(res.Metrics, func(i, j int) bool {
  1162  				return res.Metrics[i].Name < res.Metrics[j].Name
  1163  			})
  1164  			sort.Slice(tt.expectedResponse.Metrics, func(i, j int) bool {
  1165  				return tt.expectedResponse.Metrics[i].Name < tt.expectedResponse.Metrics[j].Name
  1166  			})
  1167  			if !reflect.DeepEqual(res, tt.expectedResponse) {
  1168  				t.Errorf("got %v, expected %v", res, tt.expectedResponse)
  1169  			}
  1170  		})
  1171  	}
  1172  }