github.com/cilium/cilium@v1.16.2/pkg/health/server/server_test.go (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright Authors of Cilium
     3  
     4  package server
     5  
     6  import (
     7  	"strings"
     8  	"testing"
     9  
    10  	"github.com/prometheus/client_golang/prometheus"
    11  	"github.com/prometheus/client_golang/prometheus/testutil"
    12  	"github.com/stretchr/testify/require"
    13  
    14  	healthModels "github.com/cilium/cilium/api/v1/health/models"
    15  	"github.com/cilium/cilium/pkg/metrics"
    16  	"github.com/cilium/cilium/pkg/metrics/metric"
    17  )
    18  
    19  var sampleSingleClusterConnectivity = &healthReport{
    20  	nodes: []*healthModels.NodeStatus{
    21  		{
    22  			HealthEndpoint: &healthModels.EndpointStatus{
    23  				PrimaryAddress: &healthModels.PathStatus{
    24  					HTTP: &healthModels.ConnectivityStatus{
    25  						Latency: 212100,
    26  					},
    27  					Icmp: &healthModels.ConnectivityStatus{
    28  						Latency: 672600,
    29  					},
    30  					IP: "10.244.3.219",
    31  				},
    32  				SecondaryAddresses: []*healthModels.PathStatus{
    33  					{
    34  						HTTP: &healthModels.ConnectivityStatus{
    35  							Latency: 212101,
    36  						},
    37  						Icmp: &healthModels.ConnectivityStatus{
    38  							Latency: 672601,
    39  						},
    40  						IP: "10.244.3.220",
    41  					},
    42  				},
    43  			},
    44  			Host: &healthModels.HostStatus{
    45  				PrimaryAddress: &healthModels.PathStatus{
    46  					HTTP: &healthModels.ConnectivityStatus{
    47  						Latency: 165362,
    48  					},
    49  					Icmp: &healthModels.ConnectivityStatus{
    50  						Latency: 704179,
    51  					},
    52  					IP: "172.18.0.3",
    53  				},
    54  				SecondaryAddresses: []*healthModels.PathStatus{
    55  					{
    56  						HTTP: &healthModels.ConnectivityStatus{
    57  							Latency: 212102,
    58  						},
    59  						Icmp: &healthModels.ConnectivityStatus{
    60  							Latency: 672602,
    61  						},
    62  						IP: "172.18.0.4",
    63  					},
    64  				},
    65  			},
    66  			Name: "kind-worker",
    67  		},
    68  	},
    69  }
    70  
    71  var sampleClustermeshConnectivity = &healthReport{
    72  	nodes: []*healthModels.NodeStatus{
    73  		{
    74  			HealthEndpoint: &healthModels.EndpointStatus{
    75  				PrimaryAddress: &healthModels.PathStatus{
    76  					HTTP: &healthModels.ConnectivityStatus{
    77  						Latency: 312100,
    78  					},
    79  					Icmp: &healthModels.ConnectivityStatus{
    80  						Latency: 772600,
    81  					},
    82  					IP: "10.244.3.219",
    83  				},
    84  				SecondaryAddresses: []*healthModels.PathStatus{
    85  					{
    86  						HTTP: &healthModels.ConnectivityStatus{
    87  							Latency: 312101,
    88  						},
    89  						Icmp: &healthModels.ConnectivityStatus{
    90  							Latency: 772601,
    91  						},
    92  						IP: "10.244.3.220",
    93  					},
    94  				},
    95  			},
    96  			Host: &healthModels.HostStatus{
    97  				PrimaryAddress: &healthModels.PathStatus{
    98  					HTTP: &healthModels.ConnectivityStatus{
    99  						Latency: 165362,
   100  					},
   101  					Icmp: &healthModels.ConnectivityStatus{
   102  						Latency: 704179,
   103  					},
   104  					IP: "172.18.0.1",
   105  				},
   106  				SecondaryAddresses: []*healthModels.PathStatus{
   107  					{
   108  						HTTP: &healthModels.ConnectivityStatus{
   109  							Latency: 312105,
   110  						},
   111  						Icmp: &healthModels.ConnectivityStatus{
   112  							Latency: 772606,
   113  						},
   114  						IP: "172.18.0.2",
   115  					},
   116  				},
   117  			},
   118  			Name: "kind-cilium-mesh-1/kind-cilium-mesh-1-worker",
   119  		},
   120  		{
   121  			HealthEndpoint: &healthModels.EndpointStatus{
   122  				PrimaryAddress: &healthModels.PathStatus{
   123  					HTTP: &healthModels.ConnectivityStatus{
   124  						Latency: 274815,
   125  					},
   126  					Icmp: &healthModels.ConnectivityStatus{
   127  						Latency: 583711,
   128  					},
   129  					IP: "10.1.2.143",
   130  				},
   131  				SecondaryAddresses: []*healthModels.PathStatus{
   132  					{
   133  						HTTP: &healthModels.ConnectivityStatus{
   134  							Latency: 212101,
   135  						},
   136  						Icmp: &healthModels.ConnectivityStatus{
   137  							Latency: 672601,
   138  						},
   139  						IP: "10.1.2.144",
   140  					},
   141  				},
   142  			},
   143  			Host: &healthModels.HostStatus{
   144  				PrimaryAddress: &healthModels.PathStatus{
   145  					HTTP: &healthModels.ConnectivityStatus{
   146  						Latency: 166101,
   147  					},
   148  					Icmp: &healthModels.ConnectivityStatus{
   149  						Latency: 635688,
   150  					},
   151  					IP: "172.18.0.3",
   152  				},
   153  				SecondaryAddresses: []*healthModels.PathStatus{
   154  					{
   155  						HTTP: &healthModels.ConnectivityStatus{
   156  							Latency: 212103,
   157  						},
   158  						Icmp: &healthModels.ConnectivityStatus{
   159  							Latency: 672603,
   160  						},
   161  						IP: "172.18.0.4",
   162  					},
   163  				},
   164  			},
   165  			Name: "kind-cilium-mesh-2/kind-cilium-mesh-2-worker",
   166  		},
   167  	},
   168  }
   169  
   170  var expectedSingleClusterMetric = map[string]string{
   171  	"cilium_node_connectivity_latency_seconds": `
   172  # HELP cilium_node_connectivity_latency_seconds The last observed latency between the current Cilium agent and other Cilium nodes in seconds
   173  # TYPE cilium_node_connectivity_latency_seconds gauge
   174  cilium_node_connectivity_latency_seconds{address_type="primary",protocol="http",source_cluster="default",source_node_name="kind-worker",target_cluster="default",target_node_ip="10.244.3.219",target_node_name="kind-worker",target_node_type="local_node",type="endpoint"} 0.0002121
   175  cilium_node_connectivity_latency_seconds{address_type="primary",protocol="http",source_cluster="default",source_node_name="kind-worker",target_cluster="default",target_node_ip="172.18.0.3",target_node_name="kind-worker",target_node_type="local_node",type="node"} 0.000165362
   176  cilium_node_connectivity_latency_seconds{address_type="primary",protocol="icmp",source_cluster="default",source_node_name="kind-worker",target_cluster="default",target_node_ip="10.244.3.219",target_node_name="kind-worker",target_node_type="local_node",type="endpoint"} 0.0006726
   177  cilium_node_connectivity_latency_seconds{address_type="primary",protocol="icmp",source_cluster="default",source_node_name="kind-worker",target_cluster="default",target_node_ip="172.18.0.3",target_node_name="kind-worker",target_node_type="local_node",type="node"} 0.000704179
   178  cilium_node_connectivity_latency_seconds{address_type="secondary",protocol="http",source_cluster="default",source_node_name="kind-worker",target_cluster="default",target_node_ip="10.244.3.220",target_node_name="kind-worker",target_node_type="local_node",type="endpoint"} 0.000212101
   179  cilium_node_connectivity_latency_seconds{address_type="secondary",protocol="http",source_cluster="default",source_node_name="kind-worker",target_cluster="default",target_node_ip="172.18.0.4",target_node_name="kind-worker",target_node_type="local_node",type="node"} 0.000212102
   180  cilium_node_connectivity_latency_seconds{address_type="secondary",protocol="icmp",source_cluster="default",source_node_name="kind-worker",target_cluster="default",target_node_ip="10.244.3.220",target_node_name="kind-worker",target_node_type="local_node",type="endpoint"} 0.000672601
   181  cilium_node_connectivity_latency_seconds{address_type="secondary",protocol="icmp",source_cluster="default",source_node_name="kind-worker",target_cluster="default",target_node_ip="172.18.0.4",target_node_name="kind-worker",target_node_type="local_node",type="node"} 0.000672602
   182  `,
   183  	"cilium_node_connectivity_status": `
   184  # HELP cilium_node_connectivity_status The last observed status of both ICMP and HTTP connectivity between the current Cilium agent and other Cilium nodes
   185  # TYPE cilium_node_connectivity_status gauge
   186  cilium_node_connectivity_status{source_cluster="default",source_node_name="kind-worker",target_cluster="default",target_node_name="kind-worker",target_node_type="local_node",type="endpoint"} 1
   187  cilium_node_connectivity_status{source_cluster="default",source_node_name="kind-worker",target_cluster="default",target_node_name="kind-worker",target_node_type="local_node",type="node"} 1
   188  `,
   189  }
   190  
   191  var expectedClustermeshMetric = map[string]string{
   192  	"cilium_node_connectivity_latency_seconds": `
   193  # HELP cilium_node_connectivity_latency_seconds The last observed latency between the current Cilium agent and other Cilium nodes in seconds
   194  # TYPE cilium_node_connectivity_latency_seconds gauge
   195  cilium_node_connectivity_latency_seconds{address_type="primary",protocol="http",source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-1",target_node_ip="10.244.3.219",target_node_name="kind-cilium-mesh-1-worker",target_node_type="local_node",type="endpoint"} 0.0003121
   196  cilium_node_connectivity_latency_seconds{address_type="primary",protocol="http",source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-1",target_node_ip="172.18.0.1",target_node_name="kind-cilium-mesh-1-worker",target_node_type="local_node",type="node"} 0.000165362
   197  cilium_node_connectivity_latency_seconds{address_type="primary",protocol="http",source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-2",target_node_ip="10.1.2.143",target_node_name="kind-cilium-mesh-2-worker",target_node_type="remote_inter_cluster",type="endpoint"} 0.000274815
   198  cilium_node_connectivity_latency_seconds{address_type="primary",protocol="http",source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-2",target_node_ip="172.18.0.3",target_node_name="kind-cilium-mesh-2-worker",target_node_type="remote_inter_cluster",type="node"} 0.000166101
   199  cilium_node_connectivity_latency_seconds{address_type="primary",protocol="icmp",source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-1",target_node_ip="10.244.3.219",target_node_name="kind-cilium-mesh-1-worker",target_node_type="local_node",type="endpoint"} 0.0007726
   200  cilium_node_connectivity_latency_seconds{address_type="primary",protocol="icmp",source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-1",target_node_ip="172.18.0.1",target_node_name="kind-cilium-mesh-1-worker",target_node_type="local_node",type="node"} 0.000704179
   201  cilium_node_connectivity_latency_seconds{address_type="primary",protocol="icmp",source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-2",target_node_ip="10.1.2.143",target_node_name="kind-cilium-mesh-2-worker",target_node_type="remote_inter_cluster",type="endpoint"} 0.000583711
   202  cilium_node_connectivity_latency_seconds{address_type="primary",protocol="icmp",source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-2",target_node_ip="172.18.0.3",target_node_name="kind-cilium-mesh-2-worker",target_node_type="remote_inter_cluster",type="node"} 0.000635688
   203  cilium_node_connectivity_latency_seconds{address_type="secondary",protocol="http",source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-1",target_node_ip="10.244.3.220",target_node_name="kind-cilium-mesh-1-worker",target_node_type="local_node",type="endpoint"} 0.000312101
   204  cilium_node_connectivity_latency_seconds{address_type="secondary",protocol="http",source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-1",target_node_ip="172.18.0.2",target_node_name="kind-cilium-mesh-1-worker",target_node_type="local_node",type="node"} 0.000312105
   205  cilium_node_connectivity_latency_seconds{address_type="secondary",protocol="http",source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-2",target_node_ip="10.1.2.144",target_node_name="kind-cilium-mesh-2-worker",target_node_type="remote_inter_cluster",type="endpoint"} 0.000212101
   206  cilium_node_connectivity_latency_seconds{address_type="secondary",protocol="http",source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-2",target_node_ip="172.18.0.4",target_node_name="kind-cilium-mesh-2-worker",target_node_type="remote_inter_cluster",type="node"} 0.000212103
   207  cilium_node_connectivity_latency_seconds{address_type="secondary",protocol="icmp",source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-1",target_node_ip="10.244.3.220",target_node_name="kind-cilium-mesh-1-worker",target_node_type="local_node",type="endpoint"} 0.000772601
   208  cilium_node_connectivity_latency_seconds{address_type="secondary",protocol="icmp",source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-1",target_node_ip="172.18.0.2",target_node_name="kind-cilium-mesh-1-worker",target_node_type="local_node",type="node"} 0.000772606
   209  cilium_node_connectivity_latency_seconds{address_type="secondary",protocol="icmp",source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-2",target_node_ip="10.1.2.144",target_node_name="kind-cilium-mesh-2-worker",target_node_type="remote_inter_cluster",type="endpoint"} 0.000672601
   210  cilium_node_connectivity_latency_seconds{address_type="secondary",protocol="icmp",source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-2",target_node_ip="172.18.0.4",target_node_name="kind-cilium-mesh-2-worker",target_node_type="remote_inter_cluster",type="node"} 0.000672603
   211  `,
   212  	"cilium_node_connectivity_status": `
   213  # HELP cilium_node_connectivity_status The last observed status of both ICMP and HTTP connectivity between the current Cilium agent and other Cilium nodes
   214  # TYPE cilium_node_connectivity_status gauge
   215  cilium_node_connectivity_status{source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-1",target_node_name="kind-cilium-mesh-1-worker",target_node_type="local_node",type="endpoint"} 1
   216  cilium_node_connectivity_status{source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-1",target_node_name="kind-cilium-mesh-1-worker",target_node_type="local_node",type="node"} 1
   217  cilium_node_connectivity_status{source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-2",target_node_name="kind-cilium-mesh-2-worker",target_node_type="remote_inter_cluster",type="endpoint"} 1
   218  cilium_node_connectivity_status{source_cluster="kind-cilium-mesh-1",source_node_name="kind-cilium-mesh-1-worker",target_cluster="kind-cilium-mesh-2",target_node_name="kind-cilium-mesh-2-worker",target_node_type="remote_inter_cluster",type="node"} 1
   219  `,
   220  }
   221  
   222  func Test_server_getClusterNodeName(t *testing.T) {
   223  	tests := []struct {
   224  		name                string
   225  		fullName            string
   226  		expectedClusterName string
   227  		expectedNodeName    string
   228  	}{
   229  		{
   230  			name:                "no cluster name",
   231  			fullName:            "k8s1",
   232  			expectedClusterName: "default",
   233  			expectedNodeName:    "k8s1",
   234  		},
   235  		{
   236  			name:                "simple full name",
   237  			fullName:            "kind-kind/worker",
   238  			expectedClusterName: "kind-kind",
   239  			expectedNodeName:    "worker",
   240  		},
   241  		{
   242  			name:                "cluster name is having slash",
   243  			fullName:            "arn:aws:eks:us-west-2:012345678910:cluster/cluster/name",
   244  			expectedClusterName: "arn:aws:eks:us-west-2:012345678910:cluster/cluster",
   245  			expectedNodeName:    "name",
   246  		},
   247  	}
   248  
   249  	for _, tt := range tests {
   250  		t.Run(tt.name, func(t *testing.T) {
   251  			clusterName, nodeName := getClusterNodeName(tt.fullName)
   252  			require.Equal(t, tt.expectedClusterName, clusterName)
   253  			require.Equal(t, tt.expectedNodeName, nodeName)
   254  		})
   255  	}
   256  }
   257  
   258  func Test_server_collectNodeConnectivityMetrics(t *testing.T) {
   259  	tests := []struct {
   260  		name           string
   261  		localStatus    *healthModels.SelfStatus
   262  		connectivity   *healthReport
   263  		metric         func() metric.WithMetadata
   264  		expectedMetric string
   265  		expectedCount  int
   266  	}{
   267  		{
   268  			name: "single cluster for cilium_node_connectivity_status",
   269  			localStatus: &healthModels.SelfStatus{
   270  				Name: "kind-worker",
   271  			},
   272  			connectivity:   sampleSingleClusterConnectivity,
   273  			metric:         func() metric.WithMetadata { return metrics.NodeConnectivityStatus },
   274  			expectedCount:  2,
   275  			expectedMetric: expectedSingleClusterMetric["cilium_node_connectivity_status"],
   276  		},
   277  		{
   278  			name: "single cluster for cilium_node_connectivity_latency_seconds",
   279  			localStatus: &healthModels.SelfStatus{
   280  				Name: "kind-worker",
   281  			},
   282  			connectivity:   sampleSingleClusterConnectivity,
   283  			metric:         func() metric.WithMetadata { return metrics.NodeConnectivityLatency },
   284  			expectedCount:  8,
   285  			expectedMetric: expectedSingleClusterMetric["cilium_node_connectivity_latency_seconds"],
   286  		},
   287  		{
   288  			name: "cluster mesh for cilium_node_connectivity_status",
   289  			localStatus: &healthModels.SelfStatus{
   290  				Name: "kind-cilium-mesh-1/kind-cilium-mesh-1-worker",
   291  			},
   292  			connectivity:   sampleClustermeshConnectivity,
   293  			metric:         func() metric.WithMetadata { return metrics.NodeConnectivityStatus },
   294  			expectedCount:  4,
   295  			expectedMetric: expectedClustermeshMetric["cilium_node_connectivity_status"],
   296  		},
   297  		{
   298  			name: "cluster mesh for cilium_node_connectivity_latency_seconds",
   299  			localStatus: &healthModels.SelfStatus{
   300  				Name: "kind-cilium-mesh-1/kind-cilium-mesh-1-worker",
   301  			},
   302  			connectivity:   sampleClustermeshConnectivity,
   303  			metric:         func() metric.WithMetadata { return metrics.NodeConnectivityLatency },
   304  			expectedCount:  16,
   305  			expectedMetric: expectedClustermeshMetric["cilium_node_connectivity_latency_seconds"],
   306  		},
   307  	}
   308  
   309  	for _, tt := range tests {
   310  		t.Run(tt.name, func(t *testing.T) {
   311  			metrics.NewLegacyMetrics()
   312  			tt.metric().SetEnabled(true)
   313  			collector := tt.metric().(prometheus.Collector)
   314  			s := &Server{
   315  				connectivity: tt.connectivity,
   316  				localStatus:  tt.localStatus,
   317  			}
   318  			s.collectNodeConnectivityMetrics()
   319  
   320  			// perform static checks such as prometheus naming convention, number of labels matching, etc
   321  			lintProblems, err := testutil.CollectAndLint(collector)
   322  			require.NoError(t, err)
   323  			require.Empty(t, lintProblems)
   324  
   325  			// check the number of metrics
   326  			count := testutil.CollectAndCount(collector)
   327  			require.Equal(t, tt.expectedCount, count)
   328  
   329  			// compare the metric output
   330  			err = testutil.CollectAndCompare(collector, strings.NewReader(tt.expectedMetric))
   331  			require.NoError(t, err)
   332  		})
   333  	}
   334  
   335  }