github.com/kiali/kiali@v1.84.0/graph/telemetry/istio/appender/response_time_test.go (about)

     1  package appender
     2  
     3  import (
     4  	"testing"
     5  	"time"
     6  
     7  	"github.com/prometheus/common/model"
     8  	"github.com/stretchr/testify/assert"
     9  
    10  	"github.com/kiali/kiali/config"
    11  	"github.com/kiali/kiali/graph"
    12  )
    13  
    14  func TestResponseTimeP95(t *testing.T) {
    15  	assert := assert.New(t)
    16  
    17  	q0 := `round(histogram_quantile(0.95, sum(rate(istio_request_duration_milliseconds_bucket{reporter="destination",destination_service_namespace="bookinfo"}[60s])) by (le,source_cluster,source_workload_namespace,source_workload,source_canonical_service,source_canonical_revision,destination_cluster,destination_service_namespace,destination_service,destination_service_name,destination_workload_namespace,destination_workload,destination_canonical_service,destination_canonical_revision,request_protocol)) > 0,0.001)`
    18  	q0m0 := model.Metric{
    19  		"source_cluster":                 config.DefaultClusterID,
    20  		"source_workload_namespace":      "istio-system",
    21  		"source_workload":                "ingressgateway-unknown",
    22  		"source_canonical_service":       "ingressgateway",
    23  		"source_canonical_revision":      model.LabelValue(graph.Unknown),
    24  		"destination_cluster":            config.DefaultClusterID,
    25  		"destination_service_namespace":  "bookinfo",
    26  		"destination_service":            "productpage.bookinfo.svc.cluster.local",
    27  		"destination_service_name":       "productpage",
    28  		"destination_workload_namespace": "bookinfo",
    29  		"destination_workload":           "productpage-v1",
    30  		"destination_canonical_service":  "productpage",
    31  		"destination_canonical_revision": "v1",
    32  		"request_protocol":               "http"}
    33  	q0m1 := model.Metric{
    34  		"source_cluster":                 config.DefaultClusterID,
    35  		"source_workload_namespace":      "bookinfo",
    36  		"source_workload":                "productpage-v1",
    37  		"source_canonical_service":       "productpage",
    38  		"source_canonical_revision":      "v1",
    39  		"destination_cluster":            config.DefaultClusterID,
    40  		"destination_service_namespace":  "bookinfo",
    41  		"destination_service":            "reviews.bookinfo.svc.cluster.local",
    42  		"destination_service_name":       "reviews",
    43  		"destination_workload_namespace": "bookinfo",
    44  		"destination_workload":           "reviews-v1",
    45  		"destination_canonical_service":  "reviews",
    46  		"destination_canonical_revision": "v1",
    47  		"request_protocol":               "http"}
    48  	q0m2 := model.Metric{
    49  		"source_cluster":                 config.DefaultClusterID,
    50  		"source_workload_namespace":      "bookinfo",
    51  		"source_workload":                "productpage-v1",
    52  		"source_canonical_service":       "productpage",
    53  		"source_canonical_revision":      "v1",
    54  		"destination_cluster":            config.DefaultClusterID,
    55  		"destination_service_namespace":  "bookinfo",
    56  		"destination_service":            "reviews.bookinfo.svc.cluster.local",
    57  		"destination_service_name":       "reviews",
    58  		"destination_workload_namespace": "bookinfo",
    59  		"destination_workload":           "reviews-v2",
    60  		"destination_canonical_service":  "reviews",
    61  		"destination_canonical_revision": "v2",
    62  		"request_protocol":               "http"}
    63  	q0m3 := model.Metric{
    64  		"source_cluster":                 config.DefaultClusterID,
    65  		"source_workload_namespace":      "bookinfo",
    66  		"source_workload":                "reviews-v1",
    67  		"source_canonical_service":       "reviews",
    68  		"source_canonical_revision":      "v1",
    69  		"destination_cluster":            config.DefaultClusterID,
    70  		"destination_service_namespace":  "bookinfo",
    71  		"destination_service":            "ratings.bookinfo.svc.cluster.local",
    72  		"destination_service_name":       "ratings",
    73  		"destination_workload_namespace": "bookinfo",
    74  		"destination_workload":           "ratings-v1",
    75  		"destination_canonical_service":  "ratings",
    76  		"destination_canonical_revision": "v1",
    77  		"request_protocol":               "http"}
    78  	q0m4 := model.Metric{
    79  		"source_cluster":                 config.DefaultClusterID,
    80  		"source_workload_namespace":      "bookinfo",
    81  		"source_workload":                "reviews-v2",
    82  		"source_canonical_service":       "reviews",
    83  		"source_canonical_revision":      "v2",
    84  		"destination_cluster":            config.DefaultClusterID,
    85  		"destination_service_namespace":  "bookinfo",
    86  		"destination_service":            "ratings.bookinfo.svc.cluster.local",
    87  		"destination_service_name":       "ratings",
    88  		"destination_workload_namespace": "bookinfo",
    89  		"destination_workload":           "ratings-v1",
    90  		"destination_canonical_service":  "ratings",
    91  		"destination_canonical_revision": "v1",
    92  		"request_protocol":               "http"}
    93  	v0 := model.Vector{
    94  		&model.Sample{
    95  			Metric: q0m0,
    96  			Value:  0.010},
    97  		&model.Sample{
    98  			Metric: q0m1,
    99  			Value:  0.020},
   100  		&model.Sample{
   101  			Metric: q0m2,
   102  			Value:  0.020},
   103  		&model.Sample{
   104  			Metric: q0m3,
   105  			Value:  0.030}, // same edge reported by outgoing (q1), this > value should be preferred
   106  		&model.Sample{
   107  			Metric: q0m4,
   108  			Value:  0.030}, // same edge reported by outgoing (q1), this > value should be preferred
   109  	}
   110  
   111  	q1 := `round(histogram_quantile(0.95, sum(rate(istio_request_duration_milliseconds_bucket{reporter="source",source_workload_namespace="bookinfo"}[60s])) by (le,source_cluster,source_workload_namespace,source_workload,source_canonical_service,source_canonical_revision,destination_cluster,destination_service_namespace,destination_service,destination_service_name,destination_workload_namespace,destination_workload,destination_canonical_service,destination_canonical_revision,request_protocol)) > 0,0.001)`
   112  	q1m0 := model.Metric{
   113  		"source_cluster":                 config.DefaultClusterID,
   114  		"source_workload_namespace":      "bookinfo",
   115  		"source_workload":                "productpage-v1",
   116  		"source_canonical_service":       "productpage",
   117  		"source_canonical_revision":      "v1",
   118  		"destination_cluster":            config.DefaultClusterID,
   119  		"destination_service_namespace":  "bookinfo",
   120  		"destination_service":            "reviews.bookinfo.svc.cluster.local",
   121  		"destination_service_name":       "reviews",
   122  		"destination_workload_namespace": "bookinfo",
   123  		"destination_workload":           "reviews-v1",
   124  		"destination_canonical_service":  "reviews",
   125  		"destination_canonical_revision": "v1",
   126  		"request_protocol":               "http"}
   127  	q1m1 := model.Metric{
   128  		"source_cluster":                 config.DefaultClusterID,
   129  		"source_workload_namespace":      "bookinfo",
   130  		"source_workload":                "productpage-v1",
   131  		"source_canonical_service":       "productpage",
   132  		"source_canonical_revision":      "v1",
   133  		"destination_cluster":            config.DefaultClusterID,
   134  		"destination_service_namespace":  "bookinfo",
   135  		"destination_service":            "reviews.bookinfo.svc.cluster.local",
   136  		"destination_service_name":       "reviews",
   137  		"destination_workload_namespace": "bookinfo",
   138  		"destination_workload":           "reviews-v2",
   139  		"destination_canonical_service":  "reviews",
   140  		"destination_canonical_revision": "v2",
   141  		"request_protocol":               "http"}
   142  	q1m2 := model.Metric{
   143  		"source_cluster":                 config.DefaultClusterID,
   144  		"source_workload_namespace":      "bookinfo",
   145  		"source_workload":                "reviews-v1",
   146  		"source_canonical_service":       "reviews",
   147  		"source_canonical_revision":      "v1",
   148  		"destination_cluster":            config.DefaultClusterID,
   149  		"destination_service_namespace":  "bookinfo",
   150  		"destination_service":            "ratings.bookinfo.svc.cluster.local",
   151  		"destination_service_name":       "ratings",
   152  		"destination_workload_namespace": "bookinfo",
   153  		"destination_workload":           "ratings-v1",
   154  		"destination_canonical_service":  "ratings",
   155  		"destination_canonical_revision": "v1",
   156  		"request_protocol":               "http"}
   157  	q1m3 := model.Metric{
   158  		"source_cluster":                 config.DefaultClusterID,
   159  		"source_workload_namespace":      "bookinfo",
   160  		"source_workload":                "reviews-v2",
   161  		"source_canonical_service":       "reviews",
   162  		"source_canonical_revision":      "v2",
   163  		"destination_cluster":            config.DefaultClusterID,
   164  		"destination_service_namespace":  "bookinfo",
   165  		"destination_service":            "ratings.bookinfo.svc.cluster.local",
   166  		"destination_service_name":       "ratings",
   167  		"destination_workload_namespace": "bookinfo",
   168  		"destination_workload":           "ratings-v1",
   169  		"destination_canonical_service":  "ratings",
   170  		"destination_canonical_revision": "v1",
   171  		"request_protocol":               "http"}
   172  
   173  	v1 := model.Vector{
   174  		&model.Sample{
   175  			Metric: q1m0,
   176  			Value:  0.020},
   177  		&model.Sample{
   178  			Metric: q1m1,
   179  			Value:  0.020},
   180  		&model.Sample{
   181  			Metric: q1m2,
   182  			Value:  0.040}, // same edge reported by incoming (q0), this > value should get ignored
   183  		&model.Sample{
   184  			Metric: q1m3,
   185  			Value:  0.040}, // same edge reported by incoming (q0), this > value should get ignored
   186  	}
   187  
   188  	client, api, err := setupMocked()
   189  	if err != nil {
   190  		t.Error(err)
   191  		return
   192  	}
   193  	mockQuery(api, q0, &v0)
   194  	mockQuery(api, q1, &v1)
   195  
   196  	trafficMap := responseTimeTestTraffic()
   197  	ingressID, _, _ := graph.Id(config.DefaultClusterID, "istio-system", "", "istio-system", "ingressgateway-unknown", "ingressgateway", graph.Unknown, graph.GraphTypeVersionedApp)
   198  	ingress, ok := trafficMap[ingressID]
   199  	assert.Equal(true, ok)
   200  	assert.Equal("ingressgateway", ingress.App)
   201  	assert.Equal(1, len(ingress.Edges))
   202  	assert.Equal(nil, ingress.Edges[0].Metadata[graph.ResponseTime])
   203  
   204  	duration, _ := time.ParseDuration("60s")
   205  	appender := ResponseTimeAppender{
   206  		GraphType:          graph.GraphTypeVersionedApp,
   207  		InjectServiceNodes: true,
   208  		Namespaces: map[string]graph.NamespaceInfo{
   209  			"bookinfo": {
   210  				Name:     "bookinfo",
   211  				Duration: duration,
   212  			},
   213  		},
   214  		Quantile:  0.95,
   215  		QueryTime: time.Now().Unix(),
   216  		Rates: graph.RequestedRates{
   217  			Grpc: graph.RateRequests,
   218  			Http: graph.RateRequests,
   219  			Tcp:  graph.RateTotal,
   220  		},
   221  	}
   222  
   223  	appender.appendGraph(trafficMap, "bookinfo", client)
   224  
   225  	ingress, ok = trafficMap[ingressID]
   226  	assert.Equal(true, ok)
   227  	assert.Equal("ingressgateway", ingress.App)
   228  	assert.Equal(1, len(ingress.Edges))
   229  	_, ok = ingress.Edges[0].Metadata[graph.ResponseTime]
   230  	assert.Equal(false, ok)
   231  
   232  	productpageService := ingress.Edges[0].Dest
   233  	assert.Equal(graph.NodeTypeService, productpageService.NodeType)
   234  	assert.Equal("productpage", productpageService.Service)
   235  	assert.Equal(nil, productpageService.Metadata[graph.ResponseTime])
   236  	assert.Equal(1, len(productpageService.Edges))
   237  	assert.Equal(0.01, productpageService.Edges[0].Metadata[graph.ResponseTime])
   238  
   239  	productpage := productpageService.Edges[0].Dest
   240  	assert.Equal("productpage", productpage.App)
   241  	assert.Equal("v1", productpage.Version)
   242  	assert.Equal(nil, productpage.Metadata[graph.ResponseTime])
   243  	assert.Equal(1, len(productpage.Edges))
   244  	_, ok = productpage.Edges[0].Metadata[graph.ResponseTime]
   245  	assert.Equal(false, ok)
   246  
   247  	reviewsService := productpage.Edges[0].Dest
   248  	assert.Equal(graph.NodeTypeService, reviewsService.NodeType)
   249  	assert.Equal("reviews", reviewsService.Service)
   250  	assert.Equal(nil, reviewsService.Metadata[graph.ResponseTime])
   251  	assert.Equal(2, len(reviewsService.Edges))
   252  	assert.Equal(0.02, reviewsService.Edges[0].Metadata[graph.ResponseTime])
   253  	assert.Equal(0.02, reviewsService.Edges[1].Metadata[graph.ResponseTime])
   254  
   255  	reviews1 := reviewsService.Edges[0].Dest
   256  	assert.Equal("reviews", reviews1.App)
   257  	assert.Equal("v1", reviews1.Version)
   258  	assert.Equal(nil, reviews1.Metadata[graph.ResponseTime])
   259  	assert.Equal(1, len(reviews1.Edges))
   260  	_, ok = reviews1.Edges[0].Metadata[graph.ResponseTime]
   261  	assert.Equal(false, ok)
   262  
   263  	ratingsService := reviews1.Edges[0].Dest
   264  	assert.Equal(graph.NodeTypeService, ratingsService.NodeType)
   265  	assert.Equal("ratings", ratingsService.Service)
   266  	assert.Equal(nil, ratingsService.Metadata[graph.ResponseTime])
   267  	assert.Equal(1, len(ratingsService.Edges))
   268  	assert.Equal(0.03, ratingsService.Edges[0].Metadata[graph.ResponseTime])
   269  
   270  	reviews2 := reviewsService.Edges[1].Dest
   271  	assert.Equal("reviews", reviews2.App)
   272  	assert.Equal("v2", reviews2.Version)
   273  	assert.Equal(nil, reviews2.Metadata[graph.ResponseTime])
   274  	assert.Equal(1, len(reviews2.Edges))
   275  	_, ok = reviews2.Edges[0].Metadata[graph.ResponseTime]
   276  	assert.False(ok)
   277  
   278  	assert.Equal(ratingsService, reviews2.Edges[0].Dest)
   279  
   280  	ratings := ratingsService.Edges[0].Dest
   281  	assert.Equal("ratings", ratings.App)
   282  	assert.Equal("v1", ratings.Version)
   283  	assert.Equal(nil, ratings.Metadata[graph.ResponseTime])
   284  	assert.Equal(0, len(ratings.Edges))
   285  }
   286  
   287  func TestResponseTimeAvgSkipRates(t *testing.T) {
   288  	assert := assert.New(t)
   289  
   290  	q0 := `round(sum(rate(istio_request_duration_milliseconds_sum{reporter="destination",destination_service_namespace="bookinfo"}[60s])) by (source_cluster,source_workload_namespace,source_workload,source_canonical_service,source_canonical_revision,destination_cluster,destination_service_namespace,destination_service,destination_service_name,destination_workload_namespace,destination_workload,destination_canonical_service,destination_canonical_revision,request_protocol) / sum(rate(istio_request_duration_milliseconds_count{reporter="destination",destination_service_namespace="bookinfo"}[60s])) by (source_cluster,source_workload_namespace,source_workload,source_canonical_service,source_canonical_revision,destination_cluster,destination_service_namespace,destination_service,destination_service_name,destination_workload_namespace,destination_workload,destination_canonical_service,destination_canonical_revision,request_protocol) > 0,0.001)`
   291  	q0m0 := model.Metric{
   292  		"source_cluster":                 config.DefaultClusterID,
   293  		"source_workload_namespace":      "istio-system",
   294  		"source_workload":                "ingressgateway-unknown",
   295  		"source_canonical_service":       "ingressgateway",
   296  		"source_canonical_revision":      model.LabelValue(graph.Unknown),
   297  		"destination_cluster":            config.DefaultClusterID,
   298  		"destination_service_namespace":  "bookinfo",
   299  		"destination_service":            "productpage.bookinfo.svc.cluster.local",
   300  		"destination_service_name":       "productpage",
   301  		"destination_workload_namespace": "bookinfo",
   302  		"destination_workload":           "productpage-v1",
   303  		"destination_canonical_service":  "productpage",
   304  		"destination_canonical_revision": "v1",
   305  		"request_protocol":               "http"}
   306  	q0m1 := model.Metric{
   307  		"source_cluster":                 config.DefaultClusterID,
   308  		"source_workload_namespace":      "bookinfo",
   309  		"source_workload":                "productpage-v1",
   310  		"source_canonical_service":       "productpage",
   311  		"source_canonical_revision":      "v1",
   312  		"destination_cluster":            config.DefaultClusterID,
   313  		"destination_service_namespace":  "bookinfo",
   314  		"destination_service":            "reviews.bookinfo.svc.cluster.local",
   315  		"destination_service_name":       "reviews",
   316  		"destination_workload_namespace": "bookinfo",
   317  		"destination_workload":           "reviews-v1",
   318  		"destination_canonical_service":  "reviews",
   319  		"destination_canonical_revision": "v1",
   320  		"request_protocol":               "http"}
   321  	q0m2 := model.Metric{
   322  		"source_cluster":                 config.DefaultClusterID,
   323  		"source_workload_namespace":      "bookinfo",
   324  		"source_workload":                "productpage-v1",
   325  		"source_canonical_service":       "productpage",
   326  		"source_canonical_revision":      "v1",
   327  		"destination_cluster":            config.DefaultClusterID,
   328  		"destination_service_namespace":  "bookinfo",
   329  		"destination_service":            "reviews.bookinfo.svc.cluster.local",
   330  		"destination_service_name":       "reviews",
   331  		"destination_workload_namespace": "bookinfo",
   332  		"destination_workload":           "reviews-v2",
   333  		"destination_canonical_service":  "reviews",
   334  		"destination_canonical_revision": "v2",
   335  		"request_protocol":               "http"}
   336  	q0m3 := model.Metric{
   337  		"source_cluster":                 config.DefaultClusterID,
   338  		"source_workload_namespace":      "bookinfo",
   339  		"source_workload":                "reviews-v1",
   340  		"source_canonical_service":       "reviews",
   341  		"source_canonical_revision":      "v1",
   342  		"destination_cluster":            config.DefaultClusterID,
   343  		"destination_service_namespace":  "bookinfo",
   344  		"destination_service":            "ratings.bookinfo.svc.cluster.local",
   345  		"destination_service_name":       "ratings",
   346  		"destination_workload_namespace": "bookinfo",
   347  		"destination_workload":           "ratings-v1",
   348  		"destination_canonical_service":  "ratings",
   349  		"destination_canonical_revision": "v1",
   350  		"request_protocol":               "http"}
   351  	q0m4 := model.Metric{
   352  		"source_cluster":                 config.DefaultClusterID,
   353  		"source_workload_namespace":      "bookinfo",
   354  		"source_workload":                "reviews-v2",
   355  		"source_canonical_service":       "reviews",
   356  		"source_canonical_revision":      "v2",
   357  		"destination_cluster":            config.DefaultClusterID,
   358  		"destination_service_namespace":  "bookinfo",
   359  		"destination_service":            "ratings.bookinfo.svc.cluster.local",
   360  		"destination_service_name":       "ratings",
   361  		"destination_workload_namespace": "bookinfo",
   362  		"destination_workload":           "ratings-v1",
   363  		"destination_canonical_service":  "ratings",
   364  		"destination_canonical_revision": "v1",
   365  		"request_protocol":               "http"}
   366  	v0 := model.Vector{
   367  		&model.Sample{
   368  			Metric: q0m0,
   369  			Value:  0.010},
   370  		&model.Sample{
   371  			Metric: q0m1,
   372  			Value:  0.020},
   373  		&model.Sample{
   374  			Metric: q0m2,
   375  			Value:  0.020},
   376  		&model.Sample{
   377  			Metric: q0m3,
   378  			Value:  0.030}, // same edge reported by outgoing (q1), this > value should be preferred
   379  		&model.Sample{
   380  			Metric: q0m4,
   381  			Value:  0.030}, // same edge reported by outgoing (q1), this > value should be preferred
   382  	}
   383  
   384  	q1 := `round(sum(rate(istio_request_duration_milliseconds_sum{reporter="source",source_workload_namespace="bookinfo"}[60s])) by (source_cluster,source_workload_namespace,source_workload,source_canonical_service,source_canonical_revision,destination_cluster,destination_service_namespace,destination_service,destination_service_name,destination_workload_namespace,destination_workload,destination_canonical_service,destination_canonical_revision,request_protocol) / sum(rate(istio_request_duration_milliseconds_count{reporter="source",source_workload_namespace="bookinfo"}[60s])) by (source_cluster,source_workload_namespace,source_workload,source_canonical_service,source_canonical_revision,destination_cluster,destination_service_namespace,destination_service,destination_service_name,destination_workload_namespace,destination_workload,destination_canonical_service,destination_canonical_revision,request_protocol) > 0,0.001)`
   385  	q1m0 := model.Metric{
   386  		"source_cluster":                 config.DefaultClusterID,
   387  		"source_workload_namespace":      "bookinfo",
   388  		"source_workload":                "productpage-v1",
   389  		"source_canonical_service":       "productpage",
   390  		"source_canonical_revision":      "v1",
   391  		"destination_cluster":            config.DefaultClusterID,
   392  		"destination_service_namespace":  "bookinfo",
   393  		"destination_service":            "reviews.bookinfo.svc.cluster.local",
   394  		"destination_service_name":       "reviews",
   395  		"destination_workload_namespace": "bookinfo",
   396  		"destination_workload":           "reviews-v1",
   397  		"destination_canonical_service":  "reviews",
   398  		"destination_canonical_revision": "v1",
   399  		"request_protocol":               "http"}
   400  	q1m1 := model.Metric{
   401  		"source_cluster":                 config.DefaultClusterID,
   402  		"source_workload_namespace":      "bookinfo",
   403  		"source_workload":                "productpage-v1",
   404  		"source_canonical_service":       "productpage",
   405  		"source_canonical_revision":      "v1",
   406  		"destination_cluster":            config.DefaultClusterID,
   407  		"destination_service_namespace":  "bookinfo",
   408  		"destination_service":            "reviews.bookinfo.svc.cluster.local",
   409  		"destination_service_name":       "reviews",
   410  		"destination_workload_namespace": "bookinfo",
   411  		"destination_workload":           "reviews-v2",
   412  		"destination_canonical_service":  "reviews",
   413  		"destination_canonical_revision": "v2",
   414  		"request_protocol":               "http"}
   415  	q1m2 := model.Metric{
   416  		"source_cluster":                 config.DefaultClusterID,
   417  		"source_workload_namespace":      "bookinfo",
   418  		"source_workload":                "reviews-v1",
   419  		"source_canonical_service":       "reviews",
   420  		"source_canonical_revision":      "v1",
   421  		"destination_cluster":            config.DefaultClusterID,
   422  		"destination_service_namespace":  "bookinfo",
   423  		"destination_service":            "ratings.bookinfo.svc.cluster.local",
   424  		"destination_service_name":       "ratings",
   425  		"destination_workload_namespace": "bookinfo",
   426  		"destination_workload":           "ratings-v1",
   427  		"destination_canonical_service":  "ratings",
   428  		"destination_canonical_revision": "v1",
   429  		"request_protocol":               "http"}
   430  	q1m3 := model.Metric{
   431  		"source_cluster":                 config.DefaultClusterID,
   432  		"source_workload_namespace":      "bookinfo",
   433  		"source_workload":                "reviews-v2",
   434  		"source_canonical_service":       "reviews",
   435  		"source_canonical_revision":      "v2",
   436  		"destination_cluster":            config.DefaultClusterID,
   437  		"destination_service_namespace":  "bookinfo",
   438  		"destination_service":            "ratings.bookinfo.svc.cluster.local",
   439  		"destination_service_name":       "ratings",
   440  		"destination_workload_namespace": "bookinfo",
   441  		"destination_workload":           "ratings-v1",
   442  		"destination_canonical_service":  "ratings",
   443  		"destination_canonical_revision": "v1",
   444  		"request_protocol":               "http"}
   445  
   446  	v1 := model.Vector{
   447  		&model.Sample{
   448  			Metric: q1m0,
   449  			Value:  0.020},
   450  		&model.Sample{
   451  			Metric: q1m1,
   452  			Value:  0.020},
   453  		&model.Sample{
   454  			Metric: q1m2,
   455  			Value:  0.040}, // same edge reported by incoming (q0), this > value should get ignored
   456  		&model.Sample{
   457  			Metric: q1m3,
   458  			Value:  0.040}, // same edge reported by incoming (q0), this > value should get ignored
   459  	}
   460  
   461  	client, api, err := setupMocked()
   462  	if err != nil {
   463  		t.Error(err)
   464  		return
   465  	}
   466  	mockQuery(api, q0, &v0)
   467  	mockQuery(api, q1, &v1)
   468  
   469  	trafficMap := responseTimeTestTraffic()
   470  	ingressID, _, _ := graph.Id(config.DefaultClusterID, "istio-system", "", "istio-system", "ingressgateway-unknown", "ingressgateway", graph.Unknown, graph.GraphTypeVersionedApp)
   471  	ingress, ok := trafficMap[ingressID]
   472  	assert.Equal(true, ok)
   473  	assert.Equal("ingressgateway", ingress.App)
   474  	assert.Equal(1, len(ingress.Edges))
   475  	assert.Equal(nil, ingress.Edges[0].Metadata[graph.ResponseTime])
   476  
   477  	duration, _ := time.ParseDuration("60s")
   478  	appender := ResponseTimeAppender{
   479  		GraphType:          graph.GraphTypeVersionedApp,
   480  		InjectServiceNodes: true,
   481  		Namespaces: map[string]graph.NamespaceInfo{
   482  			"bookinfo": {
   483  				Name:     "bookinfo",
   484  				Duration: duration,
   485  			},
   486  		},
   487  		Quantile:  0.0,
   488  		QueryTime: time.Now().Unix(),
   489  		Rates: graph.RequestedRates{
   490  			Grpc: graph.RateRequests,
   491  			Http: graph.RateNone,
   492  			Tcp:  graph.RateTotal,
   493  		},
   494  	}
   495  
   496  	appender.appendGraph(trafficMap, "bookinfo", client)
   497  
   498  	ingress, ok = trafficMap[ingressID]
   499  	assert.Equal(true, ok)
   500  	assert.Equal("ingressgateway", ingress.App)
   501  	assert.Equal(1, len(ingress.Edges))
   502  	_, ok = ingress.Edges[0].Metadata[graph.ResponseTime]
   503  	assert.Equal(false, ok)
   504  
   505  	productpageService := ingress.Edges[0].Dest
   506  	assert.Equal(graph.NodeTypeService, productpageService.NodeType)
   507  	assert.Equal("productpage", productpageService.Service)
   508  	assert.Equal(nil, productpageService.Metadata[graph.ResponseTime])
   509  	assert.Equal(1, len(productpageService.Edges))
   510  	assert.Equal(nil, productpageService.Edges[0].Metadata[graph.ResponseTime])
   511  
   512  	productpage := productpageService.Edges[0].Dest
   513  	assert.Equal("productpage", productpage.App)
   514  	assert.Equal("v1", productpage.Version)
   515  	assert.Equal(nil, productpage.Metadata[graph.ResponseTime])
   516  	assert.Equal(1, len(productpage.Edges))
   517  	_, ok = productpage.Edges[0].Metadata[graph.ResponseTime]
   518  	assert.Equal(false, ok)
   519  
   520  	reviewsService := productpage.Edges[0].Dest
   521  	assert.Equal(graph.NodeTypeService, reviewsService.NodeType)
   522  	assert.Equal("reviews", reviewsService.Service)
   523  	assert.Equal(nil, reviewsService.Metadata[graph.ResponseTime])
   524  	assert.Equal(2, len(reviewsService.Edges))
   525  	assert.Equal(nil, reviewsService.Edges[0].Metadata[graph.ResponseTime])
   526  	assert.Equal(nil, reviewsService.Edges[1].Metadata[graph.ResponseTime])
   527  
   528  	reviews1 := reviewsService.Edges[0].Dest
   529  	assert.Equal("reviews", reviews1.App)
   530  	assert.Equal("v1", reviews1.Version)
   531  	assert.Equal(nil, reviews1.Metadata[graph.ResponseTime])
   532  	assert.Equal(1, len(reviews1.Edges))
   533  	_, ok = reviews1.Edges[0].Metadata[graph.ResponseTime]
   534  	assert.Equal(false, ok)
   535  
   536  	ratingsService := reviews1.Edges[0].Dest
   537  	assert.Equal(graph.NodeTypeService, ratingsService.NodeType)
   538  	assert.Equal("ratings", ratingsService.Service)
   539  	assert.Equal(nil, ratingsService.Metadata[graph.ResponseTime])
   540  	assert.Equal(1, len(ratingsService.Edges))
   541  	assert.Equal(nil, ratingsService.Edges[0].Metadata[graph.ResponseTime])
   542  
   543  	reviews2 := reviewsService.Edges[1].Dest
   544  	assert.Equal("reviews", reviews2.App)
   545  	assert.Equal("v2", reviews2.Version)
   546  	assert.Equal(nil, reviews2.Metadata[graph.ResponseTime])
   547  	assert.Equal(1, len(reviews2.Edges))
   548  	_, ok = reviews2.Edges[0].Metadata[graph.ResponseTime]
   549  	assert.False(ok)
   550  
   551  	assert.Equal(ratingsService, reviews2.Edges[0].Dest)
   552  
   553  	ratings := ratingsService.Edges[0].Dest
   554  	assert.Equal("ratings", ratings.App)
   555  	assert.Equal("v1", ratings.Version)
   556  	assert.Equal(nil, ratings.Metadata[graph.ResponseTime])
   557  	assert.Equal(0, len(ratings.Edges))
   558  }
   559  
   560  func TestResponseTimeAvg(t *testing.T) {
   561  	assert := assert.New(t)
   562  
   563  	q0 := `round(sum(rate(istio_request_duration_milliseconds_sum{reporter="destination",destination_service_namespace="bookinfo"}[60s])) by (source_cluster,source_workload_namespace,source_workload,source_canonical_service,source_canonical_revision,destination_cluster,destination_service_namespace,destination_service,destination_service_name,destination_workload_namespace,destination_workload,destination_canonical_service,destination_canonical_revision,request_protocol) / sum(rate(istio_request_duration_milliseconds_count{reporter="destination",destination_service_namespace="bookinfo"}[60s])) by (source_cluster,source_workload_namespace,source_workload,source_canonical_service,source_canonical_revision,destination_cluster,destination_service_namespace,destination_service,destination_service_name,destination_workload_namespace,destination_workload,destination_canonical_service,destination_canonical_revision,request_protocol) > 0,0.001)`
   564  	q0m0 := model.Metric{
   565  		"source_cluster":                 config.DefaultClusterID,
   566  		"source_workload_namespace":      "istio-system",
   567  		"source_workload":                "ingressgateway-unknown",
   568  		"source_canonical_service":       "ingressgateway",
   569  		"source_canonical_revision":      model.LabelValue(graph.Unknown),
   570  		"destination_cluster":            config.DefaultClusterID,
   571  		"destination_service_namespace":  "bookinfo",
   572  		"destination_service":            "productpage.bookinfo.svc.cluster.local",
   573  		"destination_service_name":       "productpage",
   574  		"destination_workload_namespace": "bookinfo",
   575  		"destination_workload":           "productpage-v1",
   576  		"destination_canonical_service":  "productpage",
   577  		"destination_canonical_revision": "v1",
   578  		"request_protocol":               "http"}
   579  	q0m1 := model.Metric{
   580  		"source_cluster":                 config.DefaultClusterID,
   581  		"source_workload_namespace":      "bookinfo",
   582  		"source_workload":                "productpage-v1",
   583  		"source_canonical_service":       "productpage",
   584  		"source_canonical_revision":      "v1",
   585  		"destination_cluster":            config.DefaultClusterID,
   586  		"destination_service_namespace":  "bookinfo",
   587  		"destination_service":            "reviews.bookinfo.svc.cluster.local",
   588  		"destination_service_name":       "reviews",
   589  		"destination_workload_namespace": "bookinfo",
   590  		"destination_workload":           "reviews-v1",
   591  		"destination_canonical_service":  "reviews",
   592  		"destination_canonical_revision": "v1",
   593  		"request_protocol":               "http"}
   594  	q0m2 := model.Metric{
   595  		"source_cluster":                 config.DefaultClusterID,
   596  		"source_workload_namespace":      "bookinfo",
   597  		"source_workload":                "productpage-v1",
   598  		"source_canonical_service":       "productpage",
   599  		"source_canonical_revision":      "v1",
   600  		"destination_cluster":            config.DefaultClusterID,
   601  		"destination_service_namespace":  "bookinfo",
   602  		"destination_service":            "reviews.bookinfo.svc.cluster.local",
   603  		"destination_service_name":       "reviews",
   604  		"destination_workload_namespace": "bookinfo",
   605  		"destination_workload":           "reviews-v2",
   606  		"destination_canonical_service":  "reviews",
   607  		"destination_canonical_revision": "v2",
   608  		"request_protocol":               "http"}
   609  	q0m3 := model.Metric{
   610  		"source_cluster":                 config.DefaultClusterID,
   611  		"source_workload_namespace":      "bookinfo",
   612  		"source_workload":                "reviews-v1",
   613  		"source_canonical_service":       "reviews",
   614  		"source_canonical_revision":      "v1",
   615  		"destination_cluster":            config.DefaultClusterID,
   616  		"destination_service_namespace":  "bookinfo",
   617  		"destination_service":            "ratings.bookinfo.svc.cluster.local",
   618  		"destination_service_name":       "ratings",
   619  		"destination_workload_namespace": "bookinfo",
   620  		"destination_workload":           "ratings-v1",
   621  		"destination_canonical_service":  "ratings",
   622  		"destination_canonical_revision": "v1",
   623  		"request_protocol":               "http"}
   624  	q0m4 := model.Metric{
   625  		"source_cluster":                 config.DefaultClusterID,
   626  		"source_workload_namespace":      "bookinfo",
   627  		"source_workload":                "reviews-v2",
   628  		"source_canonical_service":       "reviews",
   629  		"source_canonical_revision":      "v2",
   630  		"destination_cluster":            config.DefaultClusterID,
   631  		"destination_service_namespace":  "bookinfo",
   632  		"destination_service":            "ratings.bookinfo.svc.cluster.local",
   633  		"destination_service_name":       "ratings",
   634  		"destination_workload_namespace": "bookinfo",
   635  		"destination_workload":           "ratings-v1",
   636  		"destination_canonical_service":  "ratings",
   637  		"destination_canonical_revision": "v1",
   638  		"request_protocol":               "http"}
   639  	v0 := model.Vector{
   640  		&model.Sample{
   641  			Metric: q0m0,
   642  			Value:  0.010},
   643  		&model.Sample{
   644  			Metric: q0m1,
   645  			Value:  0.020},
   646  		&model.Sample{
   647  			Metric: q0m2,
   648  			Value:  0.020},
   649  		&model.Sample{
   650  			Metric: q0m3,
   651  			Value:  0.030}, // same edge reported by outgoing (q1), this > value should be preferred
   652  		&model.Sample{
   653  			Metric: q0m4,
   654  			Value:  0.030}, // same edge reported by outgoing (q1), this > value should be preferred
   655  	}
   656  
   657  	q1 := `round(sum(rate(istio_request_duration_milliseconds_sum{reporter="source",source_workload_namespace="bookinfo"}[60s])) by (source_cluster,source_workload_namespace,source_workload,source_canonical_service,source_canonical_revision,destination_cluster,destination_service_namespace,destination_service,destination_service_name,destination_workload_namespace,destination_workload,destination_canonical_service,destination_canonical_revision,request_protocol) / sum(rate(istio_request_duration_milliseconds_count{reporter="source",source_workload_namespace="bookinfo"}[60s])) by (source_cluster,source_workload_namespace,source_workload,source_canonical_service,source_canonical_revision,destination_cluster,destination_service_namespace,destination_service,destination_service_name,destination_workload_namespace,destination_workload,destination_canonical_service,destination_canonical_revision,request_protocol) > 0,0.001)`
   658  	q1m0 := model.Metric{
   659  		"source_cluster":                 config.DefaultClusterID,
   660  		"source_workload_namespace":      "bookinfo",
   661  		"source_workload":                "productpage-v1",
   662  		"source_canonical_service":       "productpage",
   663  		"source_canonical_revision":      "v1",
   664  		"destination_cluster":            config.DefaultClusterID,
   665  		"destination_service_namespace":  "bookinfo",
   666  		"destination_service":            "reviews.bookinfo.svc.cluster.local",
   667  		"destination_service_name":       "reviews",
   668  		"destination_workload_namespace": "bookinfo",
   669  		"destination_workload":           "reviews-v1",
   670  		"destination_canonical_service":  "reviews",
   671  		"destination_canonical_revision": "v1",
   672  		"request_protocol":               "http"}
   673  	q1m1 := model.Metric{
   674  		"source_cluster":                 config.DefaultClusterID,
   675  		"source_workload_namespace":      "bookinfo",
   676  		"source_workload":                "productpage-v1",
   677  		"source_canonical_service":       "productpage",
   678  		"source_canonical_revision":      "v1",
   679  		"destination_cluster":            config.DefaultClusterID,
   680  		"destination_service_namespace":  "bookinfo",
   681  		"destination_service":            "reviews.bookinfo.svc.cluster.local",
   682  		"destination_service_name":       "reviews",
   683  		"destination_workload_namespace": "bookinfo",
   684  		"destination_workload":           "reviews-v2",
   685  		"destination_canonical_service":  "reviews",
   686  		"destination_canonical_revision": "v2",
   687  		"request_protocol":               "http"}
   688  	q1m2 := model.Metric{
   689  		"source_cluster":                 config.DefaultClusterID,
   690  		"source_workload_namespace":      "bookinfo",
   691  		"source_workload":                "reviews-v1",
   692  		"source_canonical_service":       "reviews",
   693  		"source_canonical_revision":      "v1",
   694  		"destination_cluster":            config.DefaultClusterID,
   695  		"destination_service_namespace":  "bookinfo",
   696  		"destination_service":            "ratings.bookinfo.svc.cluster.local",
   697  		"destination_service_name":       "ratings",
   698  		"destination_workload_namespace": "bookinfo",
   699  		"destination_workload":           "ratings-v1",
   700  		"destination_canonical_service":  "ratings",
   701  		"destination_canonical_revision": "v1",
   702  		"request_protocol":               "http"}
   703  	q1m3 := model.Metric{
   704  		"source_cluster":                 config.DefaultClusterID,
   705  		"source_workload_namespace":      "bookinfo",
   706  		"source_workload":                "reviews-v2",
   707  		"source_canonical_service":       "reviews",
   708  		"source_canonical_revision":      "v2",
   709  		"destination_cluster":            config.DefaultClusterID,
   710  		"destination_service_namespace":  "bookinfo",
   711  		"destination_service":            "ratings.bookinfo.svc.cluster.local",
   712  		"destination_service_name":       "ratings",
   713  		"destination_workload_namespace": "bookinfo",
   714  		"destination_workload":           "ratings-v1",
   715  		"destination_canonical_service":  "ratings",
   716  		"destination_canonical_revision": "v1",
   717  		"request_protocol":               "http"}
   718  
   719  	v1 := model.Vector{
   720  		&model.Sample{
   721  			Metric: q1m0,
   722  			Value:  0.020},
   723  		&model.Sample{
   724  			Metric: q1m1,
   725  			Value:  0.020},
   726  		&model.Sample{
   727  			Metric: q1m2,
   728  			Value:  0.040}, // same edge reported by incoming (q0), this > value should get ignored
   729  		&model.Sample{
   730  			Metric: q1m3,
   731  			Value:  0.040}, // same edge reported by incoming (q0), this > value should get ignored
   732  	}
   733  
   734  	client, api, err := setupMocked()
   735  	if err != nil {
   736  		t.Error(err)
   737  		return
   738  	}
   739  	mockQuery(api, q0, &v0)
   740  	mockQuery(api, q1, &v1)
   741  
   742  	trafficMap := responseTimeTestTraffic()
   743  	ingressID, _, _ := graph.Id(config.DefaultClusterID, "istio-system", "", "istio-system", "ingressgateway-unknown", "ingressgateway", graph.Unknown, graph.GraphTypeVersionedApp)
   744  	ingress, ok := trafficMap[ingressID]
   745  	assert.Equal(true, ok)
   746  	assert.Equal("ingressgateway", ingress.App)
   747  	assert.Equal(1, len(ingress.Edges))
   748  	assert.Equal(nil, ingress.Edges[0].Metadata[graph.ResponseTime])
   749  
   750  	duration, _ := time.ParseDuration("60s")
   751  	appender := ResponseTimeAppender{
   752  		GraphType:          graph.GraphTypeVersionedApp,
   753  		InjectServiceNodes: true,
   754  		Namespaces: map[string]graph.NamespaceInfo{
   755  			"bookinfo": {
   756  				Name:     "bookinfo",
   757  				Duration: duration,
   758  			},
   759  		},
   760  		Quantile:  0.0,
   761  		QueryTime: time.Now().Unix(),
   762  		Rates: graph.RequestedRates{
   763  			Grpc: graph.RateRequests,
   764  			Http: graph.RateRequests,
   765  			Tcp:  graph.RateTotal,
   766  		},
   767  	}
   768  
   769  	appender.appendGraph(trafficMap, "bookinfo", client)
   770  
   771  	ingress, ok = trafficMap[ingressID]
   772  	assert.Equal(true, ok)
   773  	assert.Equal("ingressgateway", ingress.App)
   774  	assert.Equal(1, len(ingress.Edges))
   775  	_, ok = ingress.Edges[0].Metadata[graph.ResponseTime]
   776  	assert.Equal(false, ok)
   777  
   778  	productpageService := ingress.Edges[0].Dest
   779  	assert.Equal(graph.NodeTypeService, productpageService.NodeType)
   780  	assert.Equal("productpage", productpageService.Service)
   781  	assert.Equal(nil, productpageService.Metadata[graph.ResponseTime])
   782  	assert.Equal(1, len(productpageService.Edges))
   783  	assert.Equal(0.01, productpageService.Edges[0].Metadata[graph.ResponseTime])
   784  
   785  	productpage := productpageService.Edges[0].Dest
   786  	assert.Equal("productpage", productpage.App)
   787  	assert.Equal("v1", productpage.Version)
   788  	assert.Equal(nil, productpage.Metadata[graph.ResponseTime])
   789  	assert.Equal(1, len(productpage.Edges))
   790  	_, ok = productpage.Edges[0].Metadata[graph.ResponseTime]
   791  	assert.Equal(false, ok)
   792  
   793  	reviewsService := productpage.Edges[0].Dest
   794  	assert.Equal(graph.NodeTypeService, reviewsService.NodeType)
   795  	assert.Equal("reviews", reviewsService.Service)
   796  	assert.Equal(nil, reviewsService.Metadata[graph.ResponseTime])
   797  	assert.Equal(2, len(reviewsService.Edges))
   798  	assert.Equal(0.02, reviewsService.Edges[0].Metadata[graph.ResponseTime])
   799  	assert.Equal(0.02, reviewsService.Edges[1].Metadata[graph.ResponseTime])
   800  
   801  	reviews1 := reviewsService.Edges[0].Dest
   802  	assert.Equal("reviews", reviews1.App)
   803  	assert.Equal("v1", reviews1.Version)
   804  	assert.Equal(nil, reviews1.Metadata[graph.ResponseTime])
   805  	assert.Equal(1, len(reviews1.Edges))
   806  	_, ok = reviews1.Edges[0].Metadata[graph.ResponseTime]
   807  	assert.Equal(false, ok)
   808  
   809  	ratingsService := reviews1.Edges[0].Dest
   810  	assert.Equal(graph.NodeTypeService, ratingsService.NodeType)
   811  	assert.Equal("ratings", ratingsService.Service)
   812  	assert.Equal(nil, ratingsService.Metadata[graph.ResponseTime])
   813  	assert.Equal(1, len(ratingsService.Edges))
   814  	assert.Equal(0.03, ratingsService.Edges[0].Metadata[graph.ResponseTime])
   815  
   816  	reviews2 := reviewsService.Edges[1].Dest
   817  	assert.Equal("reviews", reviews2.App)
   818  	assert.Equal("v2", reviews2.Version)
   819  	assert.Equal(nil, reviews2.Metadata[graph.ResponseTime])
   820  	assert.Equal(1, len(reviews2.Edges))
   821  	_, ok = reviews2.Edges[0].Metadata[graph.ResponseTime]
   822  	assert.False(ok)
   823  
   824  	assert.Equal(ratingsService, reviews2.Edges[0].Dest)
   825  
   826  	ratings := ratingsService.Edges[0].Dest
   827  	assert.Equal("ratings", ratings.App)
   828  	assert.Equal("v1", ratings.Version)
   829  	assert.Equal(nil, ratings.Metadata[graph.ResponseTime])
   830  	assert.Equal(0, len(ratings.Edges))
   831  }
   832  
   833  func responseTimeTestTraffic() graph.TrafficMap {
   834  	ingress, _ := graph.NewNode(config.DefaultClusterID, "istio-system", "", "istio-system", "ingressgateway-unknown", "ingressgateway", graph.Unknown, graph.GraphTypeVersionedApp)
   835  	productpageService, _ := graph.NewNode(config.DefaultClusterID, "bookinfo", "productpage", "", "", "", "", graph.GraphTypeVersionedApp)
   836  	productpage, _ := graph.NewNode(config.DefaultClusterID, "bookinfo", "productpage", "bookinfo", "productpage-v1", "productpage", "v1", graph.GraphTypeVersionedApp)
   837  	reviewsService, _ := graph.NewNode(config.DefaultClusterID, "bookinfo", "reviews", "", "", "", "", graph.GraphTypeVersionedApp)
   838  	reviewsV1, _ := graph.NewNode(config.DefaultClusterID, "bookinfo", "reviews", "bookinfo", "reviews-v1", "reviews", "v1", graph.GraphTypeVersionedApp)
   839  	reviewsV2, _ := graph.NewNode(config.DefaultClusterID, "bookinfo", "reviews", "bookinfo", "reviews-v2", "reviews", "v2", graph.GraphTypeVersionedApp)
   840  	ratingsService, _ := graph.NewNode(config.DefaultClusterID, "bookinfo", "ratings", "", "", "", "", graph.GraphTypeVersionedApp)
   841  	ratings, _ := graph.NewNode(config.DefaultClusterID, "bookinfo", "ratings", "bookinfo", "ratings-v1", "ratings", "v1", graph.GraphTypeVersionedApp)
   842  	trafficMap := graph.NewTrafficMap()
   843  
   844  	trafficMap[ingress.ID] = ingress
   845  	trafficMap[productpageService.ID] = productpageService
   846  	trafficMap[productpage.ID] = productpage
   847  	trafficMap[reviewsService.ID] = reviewsService
   848  	trafficMap[reviewsV1.ID] = reviewsV1
   849  	trafficMap[reviewsV2.ID] = reviewsV2
   850  	trafficMap[ratingsService.ID] = ratingsService
   851  	trafficMap[ratings.ID] = ratings
   852  
   853  	ingress.AddEdge(productpageService).Metadata[graph.ProtocolKey] = "http"
   854  	productpageService.AddEdge(productpage).Metadata[graph.ProtocolKey] = "http"
   855  	productpage.AddEdge(reviewsService).Metadata[graph.ProtocolKey] = "http"
   856  	reviewsService.AddEdge(reviewsV1).Metadata[graph.ProtocolKey] = "http"
   857  	reviewsService.AddEdge(reviewsV2).Metadata[graph.ProtocolKey] = "http"
   858  	reviewsV1.AddEdge(ratingsService).Metadata[graph.ProtocolKey] = "http"
   859  	reviewsV2.AddEdge(ratingsService).Metadata[graph.ProtocolKey] = "http"
   860  	ratingsService.AddEdge(ratings).Metadata[graph.ProtocolKey] = "http"
   861  
   862  	return trafficMap
   863  }