github.com/cilium/cilium@v1.16.2/pkg/hubble/peer/handler_test.go (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright Authors of Cilium
     3  
     4  package peer
     5  
     6  import (
     7  	"net"
     8  	"sync"
     9  	"testing"
    10  
    11  	"github.com/stretchr/testify/assert"
    12  
    13  	peerpb "github.com/cilium/cilium/api/v1/peer"
    14  	"github.com/cilium/cilium/pkg/hubble/peer/serviceoption"
    15  	"github.com/cilium/cilium/pkg/node/addressing"
    16  	"github.com/cilium/cilium/pkg/node/types"
    17  )
    18  
    19  func TestNodeAdd(t *testing.T) {
    20  	tests := []struct {
    21  		name        string
    22  		withoutTLS  bool
    23  		addressPref serviceoption.AddressFamilyPreference
    24  		arg         types.Node
    25  		want        *peerpb.ChangeNotification
    26  	}{
    27  		{
    28  			name:       "node with just a name",
    29  			withoutTLS: true,
    30  			arg: types.Node{
    31  				Name: "name",
    32  			},
    33  			want: &peerpb.ChangeNotification{
    34  				Name:    "name",
    35  				Address: "",
    36  				Type:    peerpb.ChangeNotificationType_PEER_ADDED,
    37  			},
    38  		}, {
    39  			name:       "node with just a name and cluster",
    40  			withoutTLS: true,
    41  			arg: types.Node{
    42  				Name:    "name",
    43  				Cluster: "cluster",
    44  			},
    45  			want: &peerpb.ChangeNotification{
    46  				Name:    "cluster/name",
    47  				Address: "",
    48  				Type:    peerpb.ChangeNotificationType_PEER_ADDED,
    49  			},
    50  		}, {
    51  			name:        "node with name, cluster and one internal IP address",
    52  			withoutTLS:  true,
    53  			addressPref: serviceoption.AddressPreferIPv4,
    54  			arg: types.Node{
    55  				Name:    "name",
    56  				Cluster: "cluster",
    57  				IPAddresses: []types.Address{
    58  					{
    59  						Type: addressing.NodeInternalIP,
    60  						IP:   net.ParseIP("192.0.2.1"),
    61  					},
    62  				},
    63  			},
    64  			want: &peerpb.ChangeNotification{
    65  				Name:    "cluster/name",
    66  				Address: "192.0.2.1",
    67  				Type:    peerpb.ChangeNotificationType_PEER_ADDED,
    68  			},
    69  		}, {
    70  			name:        "node with name, cluster and one external IP address",
    71  			withoutTLS:  true,
    72  			addressPref: serviceoption.AddressPreferIPv4,
    73  			arg: types.Node{
    74  				Name:    "name",
    75  				Cluster: "cluster",
    76  				IPAddresses: []types.Address{
    77  					{
    78  						Type: addressing.NodeExternalIP,
    79  						IP:   net.ParseIP("192.0.2.1"),
    80  					},
    81  				},
    82  			},
    83  			want: &peerpb.ChangeNotification{
    84  				Name:    "cluster/name",
    85  				Address: "192.0.2.1",
    86  				Type:    peerpb.ChangeNotificationType_PEER_ADDED,
    87  			},
    88  		}, {
    89  			name:        "node with name, cluster and mixed IP addresses preferring IPv4",
    90  			withoutTLS:  true,
    91  			addressPref: serviceoption.AddressPreferIPv4,
    92  			arg: types.Node{
    93  				Name:    "name",
    94  				Cluster: "cluster",
    95  				IPAddresses: []types.Address{
    96  					{
    97  						Type: addressing.NodeExternalIP,
    98  						IP:   net.ParseIP("192.0.2.1"),
    99  					},
   100  					{
   101  						Type: addressing.NodeInternalIP,
   102  						IP:   net.ParseIP("fe80::1"),
   103  					},
   104  				},
   105  			},
   106  			want: &peerpb.ChangeNotification{
   107  				Name:    "cluster/name",
   108  				Address: "192.0.2.1",
   109  				Type:    peerpb.ChangeNotificationType_PEER_ADDED,
   110  			},
   111  		}, {
   112  			name:        "node with name, cluster and mixed IP addresses preferring IPv6",
   113  			withoutTLS:  true,
   114  			addressPref: serviceoption.AddressPreferIPv6,
   115  			arg: types.Node{
   116  				Name:    "name",
   117  				Cluster: "cluster",
   118  				IPAddresses: []types.Address{
   119  					{
   120  						Type: addressing.NodeExternalIP,
   121  						IP:   net.ParseIP("192.0.2.1"),
   122  					},
   123  					{
   124  						Type: addressing.NodeInternalIP,
   125  						IP:   net.ParseIP("fe80::1"),
   126  					},
   127  				},
   128  			},
   129  			want: &peerpb.ChangeNotification{
   130  				Name:    "cluster/name",
   131  				Address: "fe80::1",
   132  				Type:    peerpb.ChangeNotificationType_PEER_ADDED,
   133  			},
   134  		}, {
   135  			name: "node with a name and withTLS is set",
   136  			arg: types.Node{
   137  				Name: "name",
   138  			},
   139  			want: &peerpb.ChangeNotification{
   140  				Name:    "name",
   141  				Address: "",
   142  				Type:    peerpb.ChangeNotificationType_PEER_ADDED,
   143  				Tls: &peerpb.TLS{
   144  					ServerName: "name.default.hubble-grpc.cilium.io",
   145  				},
   146  			},
   147  		}, {
   148  			name: "node with name, cluster and withTLS is set",
   149  			arg: types.Node{
   150  				Name:    "name",
   151  				Cluster: "cluster",
   152  			},
   153  			want: &peerpb.ChangeNotification{
   154  				Name:    "cluster/name",
   155  				Address: "",
   156  				Type:    peerpb.ChangeNotificationType_PEER_ADDED,
   157  				Tls: &peerpb.TLS{
   158  					ServerName: "name.cluster.hubble-grpc.cilium.io",
   159  				},
   160  			},
   161  		}, {
   162  			name: "node name with dots",
   163  			arg: types.Node{
   164  				Name:    "my.very.long.node.name",
   165  				Cluster: "cluster",
   166  			},
   167  			want: &peerpb.ChangeNotification{
   168  				Name:    "cluster/my.very.long.node.name",
   169  				Address: "",
   170  				Type:    peerpb.ChangeNotificationType_PEER_ADDED,
   171  				Tls: &peerpb.TLS{
   172  					ServerName: "my-very-long-node-name.cluster.hubble-grpc.cilium.io",
   173  				},
   174  			},
   175  		}, {
   176  			name: "node name with dots in the cluster name section",
   177  			arg: types.Node{
   178  				Name:    "my.very.long.node.name",
   179  				Cluster: "cluster.name.with.dots",
   180  			},
   181  			want: &peerpb.ChangeNotification{
   182  				Name:    "cluster.name.with.dots/my.very.long.node.name",
   183  				Address: "",
   184  				Type:    peerpb.ChangeNotificationType_PEER_ADDED,
   185  				Tls: &peerpb.TLS{
   186  					ServerName: "my-very-long-node-name.cluster-name-with-dots.hubble-grpc.cilium.io",
   187  				},
   188  			},
   189  		},
   190  	}
   191  	for _, tt := range tests {
   192  		t.Run(tt.name, func(t *testing.T) {
   193  			h := newHandler(tt.withoutTLS, tt.addressPref, 0)
   194  			defer h.Close()
   195  
   196  			var got *peerpb.ChangeNotification
   197  			var wg sync.WaitGroup
   198  			wg.Add(1)
   199  			go func() {
   200  				got = <-h.C
   201  				wg.Done()
   202  			}()
   203  			h.NodeAdd(tt.arg)
   204  			wg.Wait()
   205  			assert.Equal(t, tt.want, got)
   206  		})
   207  	}
   208  }
   209  
   210  func TestNodeUpdate(t *testing.T) {
   211  	type args struct {
   212  		old, updated types.Node
   213  	}
   214  	tests := []struct {
   215  		name        string
   216  		withoutTLS  bool
   217  		addressPref serviceoption.AddressFamilyPreference
   218  		args        args
   219  		want        []*peerpb.ChangeNotification
   220  	}{
   221  		{
   222  			name:       "a node is renamed",
   223  			withoutTLS: true,
   224  			args: args{
   225  				types.Node{
   226  					Name: "old",
   227  				}, types.Node{
   228  					Name: "new",
   229  				}},
   230  			want: []*peerpb.ChangeNotification{
   231  				{
   232  					Name:    "old",
   233  					Address: "",
   234  					Type:    peerpb.ChangeNotificationType_PEER_DELETED,
   235  				}, {
   236  					Name:    "new",
   237  					Address: "",
   238  					Type:    peerpb.ChangeNotificationType_PEER_ADDED,
   239  				},
   240  			},
   241  		}, {
   242  			name:       "a node within a named cluster is renamed",
   243  			withoutTLS: true,
   244  			args: args{
   245  				types.Node{
   246  					Name:    "old",
   247  					Cluster: "cluster",
   248  				}, types.Node{
   249  					Name:    "new",
   250  					Cluster: "cluster",
   251  				}},
   252  			want: []*peerpb.ChangeNotification{
   253  				{
   254  					Name:    "cluster/old",
   255  					Address: "",
   256  					Type:    peerpb.ChangeNotificationType_PEER_DELETED,
   257  				}, {
   258  					Name:    "cluster/new",
   259  					Address: "",
   260  					Type:    peerpb.ChangeNotificationType_PEER_ADDED,
   261  				},
   262  			},
   263  		}, {
   264  			name:        "a node with name, cluster and one internal IP address, the latter is updated",
   265  			withoutTLS:  true,
   266  			addressPref: serviceoption.AddressPreferIPv4,
   267  			args: args{
   268  				types.Node{
   269  					Name:    "name",
   270  					Cluster: "cluster",
   271  					IPAddresses: []types.Address{
   272  						{
   273  							Type: addressing.NodeInternalIP,
   274  							IP:   net.ParseIP("192.0.2.1"),
   275  						},
   276  					},
   277  				}, types.Node{
   278  					Name:    "name",
   279  					Cluster: "cluster",
   280  					IPAddresses: []types.Address{
   281  						{
   282  							Type: addressing.NodeInternalIP,
   283  							IP:   net.ParseIP("192.0.2.2"),
   284  						},
   285  					}},
   286  			},
   287  			want: []*peerpb.ChangeNotification{
   288  				{
   289  					Name:    "cluster/name",
   290  					Address: "192.0.2.2",
   291  					Type:    peerpb.ChangeNotificationType_PEER_UPDATED,
   292  				},
   293  			},
   294  		}, {
   295  			name:        "node with name, cluster and one external IP address, the latter is updated",
   296  			withoutTLS:  true,
   297  			addressPref: serviceoption.AddressPreferIPv4,
   298  			args: args{
   299  				types.Node{
   300  					Name:    "name",
   301  					Cluster: "cluster",
   302  					IPAddresses: []types.Address{
   303  						{
   304  							Type: addressing.NodeExternalIP,
   305  							IP:   net.ParseIP("192.0.2.1"),
   306  						},
   307  					},
   308  				}, types.Node{
   309  					Name:    "name",
   310  					Cluster: "cluster",
   311  					IPAddresses: []types.Address{
   312  						{
   313  							Type: addressing.NodeExternalIP,
   314  							IP:   net.ParseIP("192.0.2.2"),
   315  						},
   316  					},
   317  				}},
   318  			want: []*peerpb.ChangeNotification{
   319  				{
   320  					Name:    "cluster/name",
   321  					Address: "192.0.2.2",
   322  					Type:    peerpb.ChangeNotificationType_PEER_UPDATED,
   323  				},
   324  			},
   325  		}, {
   326  			name:        "node with name, cluster and mixed IP addresses preferring IPv4, the latter is updated",
   327  			withoutTLS:  true,
   328  			addressPref: serviceoption.AddressPreferIPv4,
   329  			args: args{
   330  				types.Node{
   331  					Name:    "name",
   332  					Cluster: "cluster",
   333  					IPAddresses: []types.Address{
   334  						{
   335  							Type: addressing.NodeExternalIP,
   336  							IP:   net.ParseIP("192.0.2.1"),
   337  						},
   338  					},
   339  				}, types.Node{
   340  					Name:    "name",
   341  					Cluster: "cluster",
   342  					IPAddresses: []types.Address{
   343  						{
   344  							Type: addressing.NodeExternalIP,
   345  							IP:   net.ParseIP("192.0.2.2"),
   346  						},
   347  						{
   348  							Type: addressing.NodeInternalIP,
   349  							IP:   net.ParseIP("fe80::2"),
   350  						},
   351  					},
   352  				}},
   353  			want: []*peerpb.ChangeNotification{
   354  				{
   355  					Name:    "cluster/name",
   356  					Address: "192.0.2.2",
   357  					Type:    peerpb.ChangeNotificationType_PEER_UPDATED,
   358  				},
   359  			},
   360  		}, {
   361  			name:        "node with name, cluster and mixed IP addresses preferring IPv6, the latter is updated",
   362  			withoutTLS:  true,
   363  			addressPref: serviceoption.AddressPreferIPv6,
   364  			args: args{
   365  				types.Node{
   366  					Name:    "name",
   367  					Cluster: "cluster",
   368  					IPAddresses: []types.Address{
   369  						{
   370  							Type: addressing.NodeExternalIP,
   371  							IP:   net.ParseIP("fe80::1"),
   372  						},
   373  					},
   374  				}, types.Node{
   375  					Name:    "name",
   376  					Cluster: "cluster",
   377  					IPAddresses: []types.Address{
   378  						{
   379  							Type: addressing.NodeExternalIP,
   380  							IP:   net.ParseIP("192.0.2.2"),
   381  						},
   382  						{
   383  							Type: addressing.NodeInternalIP,
   384  							IP:   net.ParseIP("fe80::2"),
   385  						},
   386  					},
   387  				}},
   388  			want: []*peerpb.ChangeNotification{
   389  				{
   390  					Name:    "cluster/name",
   391  					Address: "fe80::2",
   392  					Type:    peerpb.ChangeNotificationType_PEER_UPDATED,
   393  				},
   394  			},
   395  		}, {
   396  			name:       "node with name, cluster and one external IP address, no name or address change",
   397  			withoutTLS: true,
   398  			args: args{
   399  				types.Node{
   400  					Name:    "name",
   401  					Cluster: "cluster",
   402  					IPAddresses: []types.Address{
   403  						{
   404  							Type: addressing.NodeExternalIP,
   405  							IP:   net.ParseIP("192.0.2.1"),
   406  						},
   407  					},
   408  				}, types.Node{
   409  					Name:    "name",
   410  					Cluster: "cluster",
   411  					IPAddresses: []types.Address{
   412  						{
   413  							Type: addressing.NodeExternalIP,
   414  							IP:   net.ParseIP("192.0.2.1"),
   415  						},
   416  					},
   417  				}},
   418  			want: nil,
   419  		}, {
   420  			name: "a node is renamed and withTLS is set",
   421  			args: args{
   422  				types.Node{
   423  					Name: "old",
   424  				}, types.Node{
   425  					Name: "new",
   426  				}},
   427  			want: []*peerpb.ChangeNotification{
   428  				{
   429  					Name:    "old",
   430  					Address: "",
   431  					Type:    peerpb.ChangeNotificationType_PEER_DELETED,
   432  					Tls: &peerpb.TLS{
   433  						ServerName: "old.default.hubble-grpc.cilium.io",
   434  					},
   435  				}, {
   436  					Name:    "new",
   437  					Address: "",
   438  					Type:    peerpb.ChangeNotificationType_PEER_ADDED,
   439  					Tls: &peerpb.TLS{
   440  						ServerName: "new.default.hubble-grpc.cilium.io",
   441  					},
   442  				},
   443  			},
   444  		}, {
   445  			name: "a node within a named cluster is renamed and withTLS is set",
   446  			args: args{
   447  				types.Node{
   448  					Name:    "old",
   449  					Cluster: "cluster",
   450  				}, types.Node{
   451  					Name:    "new",
   452  					Cluster: "cluster",
   453  				}},
   454  			want: []*peerpb.ChangeNotification{
   455  				{
   456  					Name:    "cluster/old",
   457  					Address: "",
   458  					Type:    peerpb.ChangeNotificationType_PEER_DELETED,
   459  					Tls: &peerpb.TLS{
   460  						ServerName: "old.cluster.hubble-grpc.cilium.io",
   461  					},
   462  				}, {
   463  					Name:    "cluster/new",
   464  					Address: "",
   465  					Type:    peerpb.ChangeNotificationType_PEER_ADDED,
   466  					Tls: &peerpb.TLS{
   467  						ServerName: "new.cluster.hubble-grpc.cilium.io",
   468  					},
   469  				},
   470  			},
   471  		},
   472  	}
   473  	for _, tt := range tests {
   474  		t.Run(tt.name, func(t *testing.T) {
   475  			h := newHandler(tt.withoutTLS, tt.addressPref, 0)
   476  			defer h.Close()
   477  
   478  			var got []*peerpb.ChangeNotification
   479  			var wg sync.WaitGroup
   480  			wg.Add(1)
   481  			go func() {
   482  				for i := 0; i < len(tt.want); i++ {
   483  					got = append(got, <-h.C)
   484  				}
   485  				wg.Done()
   486  			}()
   487  			h.NodeUpdate(tt.args.old, tt.args.updated)
   488  			wg.Wait()
   489  			assert.Equal(t, tt.want, got)
   490  		})
   491  	}
   492  }
   493  
   494  func TestNodeDelete(t *testing.T) {
   495  	tests := []struct {
   496  		name        string
   497  		withoutTLS  bool
   498  		addressPref serviceoption.AddressFamilyPreference
   499  		arg         types.Node
   500  		want        *peerpb.ChangeNotification
   501  	}{
   502  		{
   503  			name:       "node with just a name",
   504  			withoutTLS: true,
   505  			arg: types.Node{
   506  				Name: "name",
   507  			},
   508  			want: &peerpb.ChangeNotification{
   509  				Name:    "name",
   510  				Address: "",
   511  				Type:    peerpb.ChangeNotificationType_PEER_DELETED,
   512  			},
   513  		}, {
   514  			name:       "node with just a name and cluster",
   515  			withoutTLS: true,
   516  			arg: types.Node{
   517  				Name:    "name",
   518  				Cluster: "cluster",
   519  			},
   520  			want: &peerpb.ChangeNotification{
   521  				Name:    "cluster/name",
   522  				Address: "",
   523  				Type:    peerpb.ChangeNotificationType_PEER_DELETED,
   524  			},
   525  		}, {
   526  			name:        "node with name, cluster and one internal IP address",
   527  			withoutTLS:  true,
   528  			addressPref: serviceoption.AddressPreferIPv4,
   529  			arg: types.Node{
   530  				Name:    "name",
   531  				Cluster: "cluster",
   532  				IPAddresses: []types.Address{
   533  					{
   534  						Type: addressing.NodeInternalIP,
   535  						IP:   net.ParseIP("192.0.2.1"),
   536  					},
   537  				},
   538  			},
   539  			want: &peerpb.ChangeNotification{
   540  				Name:    "cluster/name",
   541  				Address: "192.0.2.1",
   542  				Type:    peerpb.ChangeNotificationType_PEER_DELETED,
   543  			},
   544  		}, {
   545  			name:        "node with name, cluster and one external IP address",
   546  			withoutTLS:  true,
   547  			addressPref: serviceoption.AddressPreferIPv4,
   548  			arg: types.Node{
   549  				Name:    "name",
   550  				Cluster: "cluster",
   551  				IPAddresses: []types.Address{
   552  					{
   553  						Type: addressing.NodeExternalIP,
   554  						IP:   net.ParseIP("192.0.2.1"),
   555  					},
   556  				},
   557  			},
   558  			want: &peerpb.ChangeNotification{
   559  				Name:    "cluster/name",
   560  				Address: "192.0.2.1",
   561  				Type:    peerpb.ChangeNotificationType_PEER_DELETED,
   562  			},
   563  		}, {
   564  			name:        "node with name, cluster and mixed IP addresses preferring IPv4",
   565  			withoutTLS:  true,
   566  			addressPref: serviceoption.AddressPreferIPv4,
   567  			arg: types.Node{
   568  				Name:    "name",
   569  				Cluster: "cluster",
   570  				IPAddresses: []types.Address{
   571  					{
   572  						Type: addressing.NodeExternalIP,
   573  						IP:   net.ParseIP("192.0.2.1"),
   574  					},
   575  					{
   576  						Type: addressing.NodeInternalIP,
   577  						IP:   net.ParseIP("fe80::1"),
   578  					},
   579  				},
   580  			},
   581  			want: &peerpb.ChangeNotification{
   582  				Name:    "cluster/name",
   583  				Address: "192.0.2.1",
   584  				Type:    peerpb.ChangeNotificationType_PEER_DELETED,
   585  			},
   586  		}, {
   587  			name:        "node with name, cluster and mixed IP addresses preferring IPv6",
   588  			withoutTLS:  true,
   589  			addressPref: serviceoption.AddressPreferIPv6,
   590  			arg: types.Node{
   591  				Name:    "name",
   592  				Cluster: "cluster",
   593  				IPAddresses: []types.Address{
   594  					{
   595  						Type: addressing.NodeExternalIP,
   596  						IP:   net.ParseIP("192.0.2.1"),
   597  					},
   598  					{
   599  						Type: addressing.NodeInternalIP,
   600  						IP:   net.ParseIP("fe80::1"),
   601  					},
   602  				},
   603  			},
   604  			want: &peerpb.ChangeNotification{
   605  				Name:    "cluster/name",
   606  				Address: "fe80::1",
   607  				Type:    peerpb.ChangeNotificationType_PEER_DELETED,
   608  			},
   609  		}, {
   610  			name: "node with a name and withTLS is set",
   611  			arg: types.Node{
   612  				Name: "name",
   613  			},
   614  			want: &peerpb.ChangeNotification{
   615  				Name:    "name",
   616  				Address: "",
   617  				Type:    peerpb.ChangeNotificationType_PEER_DELETED,
   618  				Tls: &peerpb.TLS{
   619  					ServerName: "name.default.hubble-grpc.cilium.io",
   620  				},
   621  			},
   622  		}, {
   623  			name: "node with a name and cluster and withTLS is set",
   624  			arg: types.Node{
   625  				Name:    "name",
   626  				Cluster: "cluster",
   627  			},
   628  			want: &peerpb.ChangeNotification{
   629  				Name:    "cluster/name",
   630  				Address: "",
   631  				Type:    peerpb.ChangeNotificationType_PEER_DELETED,
   632  				Tls: &peerpb.TLS{
   633  					ServerName: "name.cluster.hubble-grpc.cilium.io",
   634  				},
   635  			},
   636  		},
   637  	}
   638  	for _, tt := range tests {
   639  		t.Run(tt.name, func(t *testing.T) {
   640  			h := newHandler(tt.withoutTLS, tt.addressPref, 0)
   641  			defer h.Close()
   642  
   643  			var got *peerpb.ChangeNotification
   644  			var wg sync.WaitGroup
   645  			wg.Add(1)
   646  			go func() {
   647  				got = <-h.C
   648  				wg.Done()
   649  			}()
   650  			h.NodeDelete(tt.arg)
   651  			wg.Wait()
   652  			assert.Equal(t, tt.want, got)
   653  		})
   654  	}
   655  }
   656  
   657  func TestHubblePort(t *testing.T) {
   658  	tests := []struct {
   659  		name        string
   660  		addressPref serviceoption.AddressFamilyPreference
   661  		arg         types.Node
   662  		want        string
   663  	}{
   664  		{
   665  			name: "no ip",
   666  			arg:  types.Node{},
   667  			want: "",
   668  		}, {
   669  			name:        "ipv4",
   670  			addressPref: serviceoption.AddressPreferIPv4,
   671  			arg: types.Node{
   672  				IPAddresses: []types.Address{
   673  					{
   674  						Type: addressing.NodeExternalIP,
   675  						IP:   net.ParseIP("192.0.2.1"),
   676  					},
   677  				},
   678  			},
   679  			want: "192.0.2.1:9876",
   680  		}, {
   681  			name:        "ipv6",
   682  			addressPref: serviceoption.AddressPreferIPv6,
   683  			arg: types.Node{
   684  				IPAddresses: []types.Address{
   685  					{
   686  						Type: addressing.NodeInternalIP,
   687  						IP:   net.ParseIP("fe80::1"),
   688  					},
   689  				},
   690  			},
   691  			want: "[fe80::1]:9876",
   692  		},
   693  	}
   694  	for _, tt := range tests {
   695  		t.Run(tt.name, func(t *testing.T) {
   696  			h := newHandler(true, tt.addressPref, 9876)
   697  			defer h.Close()
   698  
   699  			var got *peerpb.ChangeNotification
   700  			var wg sync.WaitGroup
   701  			wg.Add(1)
   702  			go func() {
   703  				got = <-h.C
   704  				wg.Done()
   705  			}()
   706  			h.NodeAdd(tt.arg)
   707  
   708  			want := &peerpb.ChangeNotification{
   709  				Address: tt.want,
   710  				Type:    peerpb.ChangeNotificationType_PEER_ADDED,
   711  			}
   712  			wg.Wait()
   713  			assert.Equal(t, want, got)
   714  		})
   715  	}
   716  }