github.com/cilium/cilium@v1.16.2/pkg/hubble/relay/observer/server_test.go (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright Authors of Cilium
     3  
     4  package observer
     5  
     6  import (
     7  	"bytes"
     8  	"context"
     9  	"fmt"
    10  	"io"
    11  	"net"
    12  	"sync/atomic"
    13  	"testing"
    14  	"time"
    15  
    16  	"github.com/google/go-cmp/cmp"
    17  	"github.com/google/go-cmp/cmp/cmpopts"
    18  	"github.com/sirupsen/logrus"
    19  	"github.com/stretchr/testify/assert"
    20  	"github.com/stretchr/testify/require"
    21  	"google.golang.org/grpc"
    22  	"google.golang.org/grpc/codes"
    23  	"google.golang.org/grpc/connectivity"
    24  	"google.golang.org/grpc/status"
    25  	"google.golang.org/protobuf/types/known/wrapperspb"
    26  
    27  	flowpb "github.com/cilium/cilium/api/v1/flow"
    28  	observerpb "github.com/cilium/cilium/api/v1/observer"
    29  	relaypb "github.com/cilium/cilium/api/v1/relay"
    30  	"github.com/cilium/cilium/pkg/hubble/defaults"
    31  	peerTypes "github.com/cilium/cilium/pkg/hubble/peer/types"
    32  	poolTypes "github.com/cilium/cilium/pkg/hubble/relay/pool/types"
    33  	"github.com/cilium/cilium/pkg/hubble/testutils"
    34  )
    35  
    36  func TestGetFlows(t *testing.T) {
    37  	type results struct {
    38  		numFlows     int
    39  		flows        map[string][]*flowpb.Flow
    40  		statusEvents []*relaypb.NodeStatusEvent
    41  	}
    42  	var got *results
    43  	type want struct {
    44  		flows        map[string][]*flowpb.Flow
    45  		statusEvents []*relaypb.NodeStatusEvent
    46  		err          error
    47  		log          []string
    48  	}
    49  	fss := &testutils.FakeGRPCServerStream{
    50  		OnContext: context.TODO,
    51  	}
    52  	done := make(chan struct{})
    53  	tests := []struct {
    54  		name   string
    55  		plr    PeerLister
    56  		ocb    observerClientBuilder
    57  		req    *observerpb.GetFlowsRequest
    58  		stream observerpb.Observer_GetFlowsServer
    59  		want   want
    60  	}{
    61  		{
    62  			name: "Observe 0 flows from 1 peer without address",
    63  			plr: &testutils.FakePeerLister{
    64  				OnList: func() []poolTypes.Peer {
    65  					return []poolTypes.Peer{
    66  						{
    67  							Peer: peerTypes.Peer{
    68  								Name:    "noip",
    69  								Address: nil,
    70  							},
    71  							Conn: nil,
    72  						},
    73  					}
    74  				},
    75  			},
    76  			ocb: fakeObserverClientBuilder{},
    77  			req: &observerpb.GetFlowsRequest{Number: 0},
    78  			stream: &testutils.FakeGetFlowsServer{
    79  				FakeGRPCServerStream: fss,
    80  				OnSend: func(resp *observerpb.GetFlowsResponse) error {
    81  					if resp == nil {
    82  						return nil
    83  					}
    84  					switch resp.GetResponseTypes().(type) {
    85  					case *observerpb.GetFlowsResponse_Flow:
    86  						got.numFlows++
    87  						got.flows[resp.GetNodeName()] = append(got.flows[resp.GetNodeName()], resp.GetFlow())
    88  					case *observerpb.GetFlowsResponse_NodeStatus:
    89  						got.statusEvents = append(got.statusEvents, resp.GetNodeStatus())
    90  					}
    91  					if got.numFlows == 0 && len(got.statusEvents) == 1 {
    92  						close(done)
    93  						return io.EOF
    94  					}
    95  					return nil
    96  				},
    97  			},
    98  			want: want{
    99  				flows: map[string][]*flowpb.Flow{},
   100  				statusEvents: []*relaypb.NodeStatusEvent{
   101  					{
   102  						StateChange: relaypb.NodeState_NODE_UNAVAILABLE,
   103  						NodeNames:   []string{"noip"},
   104  					},
   105  				},
   106  				err: io.EOF,
   107  			},
   108  		}, {
   109  			name: "Observe 4 flows from 2 online peers",
   110  			plr: &testutils.FakePeerLister{
   111  				OnList: func() []poolTypes.Peer {
   112  					return []poolTypes.Peer{
   113  						{
   114  							Peer: peerTypes.Peer{
   115  								Name: "one",
   116  								Address: &net.TCPAddr{
   117  									IP:   net.ParseIP("192.0.2.1"),
   118  									Port: defaults.ServerPort,
   119  								},
   120  							},
   121  							Conn: &testutils.FakeClientConn{
   122  								OnGetState: func() connectivity.State {
   123  									return connectivity.Ready
   124  								},
   125  							},
   126  						}, {
   127  							Peer: peerTypes.Peer{
   128  								Name: "two",
   129  								Address: &net.TCPAddr{
   130  									IP:   net.ParseIP("192.0.2.2"),
   131  									Port: defaults.ServerPort,
   132  								},
   133  							},
   134  							Conn: &testutils.FakeClientConn{
   135  								OnGetState: func() connectivity.State {
   136  									return connectivity.Ready
   137  								},
   138  							},
   139  						},
   140  					}
   141  				},
   142  			},
   143  			ocb: fakeObserverClientBuilder{
   144  				onObserverClient: func(p *poolTypes.Peer) observerpb.ObserverClient {
   145  					var numRecv uint64
   146  					return &testutils.FakeObserverClient{
   147  						OnGetFlows: func(_ context.Context, in *observerpb.GetFlowsRequest, _ ...grpc.CallOption) (observerpb.Observer_GetFlowsClient, error) {
   148  							return &testutils.FakeGetFlowsClient{
   149  								OnRecv: func() (*observerpb.GetFlowsResponse, error) {
   150  									if numRecv == in.Number {
   151  										return nil, io.EOF
   152  									}
   153  									numRecv++
   154  									return &observerpb.GetFlowsResponse{
   155  										NodeName: p.Name,
   156  										ResponseTypes: &observerpb.GetFlowsResponse_Flow{
   157  											Flow: &flowpb.Flow{
   158  												NodeName: p.Name,
   159  											},
   160  										},
   161  									}, nil
   162  								},
   163  							}, nil
   164  						},
   165  					}
   166  				},
   167  			},
   168  			req: &observerpb.GetFlowsRequest{Number: 2},
   169  			stream: &testutils.FakeGetFlowsServer{
   170  				FakeGRPCServerStream: fss,
   171  				OnSend: func(resp *observerpb.GetFlowsResponse) error {
   172  					if resp == nil {
   173  						return nil
   174  					}
   175  					switch resp.GetResponseTypes().(type) {
   176  					case *observerpb.GetFlowsResponse_Flow:
   177  						got.numFlows++
   178  						got.flows[resp.GetNodeName()] = append(got.flows[resp.GetNodeName()], resp.GetFlow())
   179  					case *observerpb.GetFlowsResponse_NodeStatus:
   180  						got.statusEvents = append(got.statusEvents, resp.GetNodeStatus())
   181  					}
   182  					if got.numFlows == 4 && len(got.statusEvents) == 1 {
   183  						close(done)
   184  						return io.EOF
   185  					}
   186  					return nil
   187  				},
   188  			},
   189  			want: want{
   190  				flows: map[string][]*flowpb.Flow{
   191  					"one": {&flowpb.Flow{NodeName: "one"}, &flowpb.Flow{NodeName: "one"}},
   192  					"two": {&flowpb.Flow{NodeName: "two"}, &flowpb.Flow{NodeName: "two"}},
   193  				},
   194  				statusEvents: []*relaypb.NodeStatusEvent{
   195  					{
   196  						StateChange: relaypb.NodeState_NODE_CONNECTED,
   197  						NodeNames:   []string{"one", "two"},
   198  					},
   199  				},
   200  				err: io.EOF,
   201  			},
   202  		}, {
   203  			name: "Observe 2 flows from 1 online peer and none from 1 unavailable peer",
   204  			plr: &testutils.FakePeerLister{
   205  				OnList: func() []poolTypes.Peer {
   206  					return []poolTypes.Peer{
   207  						{
   208  							Peer: peerTypes.Peer{
   209  								Name: "one",
   210  								Address: &net.TCPAddr{
   211  									IP:   net.ParseIP("192.0.2.1"),
   212  									Port: defaults.ServerPort,
   213  								},
   214  							},
   215  							Conn: &testutils.FakeClientConn{
   216  								OnGetState: func() connectivity.State {
   217  									return connectivity.Ready
   218  								},
   219  							},
   220  						}, {
   221  							Peer: peerTypes.Peer{
   222  								Name: "two",
   223  								Address: &net.TCPAddr{
   224  									IP:   net.ParseIP("192.0.2.2"),
   225  									Port: defaults.ServerPort,
   226  								},
   227  							},
   228  							Conn: &testutils.FakeClientConn{
   229  								OnGetState: func() connectivity.State {
   230  									return connectivity.TransientFailure
   231  								},
   232  							},
   233  						},
   234  					}
   235  				},
   236  			},
   237  			ocb: fakeObserverClientBuilder{
   238  				onObserverClient: func(p *poolTypes.Peer) observerpb.ObserverClient {
   239  					var numRecv uint64
   240  					return &testutils.FakeObserverClient{
   241  						OnGetFlows: func(_ context.Context, in *observerpb.GetFlowsRequest, _ ...grpc.CallOption) (observerpb.Observer_GetFlowsClient, error) {
   242  							if p.Name != "one" {
   243  								return nil, fmt.Errorf("GetFlows() called for peer '%s'; this is unexpected", p.Name)
   244  							}
   245  							return &testutils.FakeGetFlowsClient{
   246  								OnRecv: func() (*observerpb.GetFlowsResponse, error) {
   247  									if numRecv == in.Number {
   248  										return nil, io.EOF
   249  									}
   250  									numRecv++
   251  									return &observerpb.GetFlowsResponse{
   252  										NodeName: p.Name,
   253  										ResponseTypes: &observerpb.GetFlowsResponse_Flow{
   254  											Flow: &flowpb.Flow{
   255  												NodeName: p.Name,
   256  											},
   257  										},
   258  									}, nil
   259  								},
   260  							}, nil
   261  						},
   262  					}
   263  				},
   264  			},
   265  			req: &observerpb.GetFlowsRequest{Number: 2},
   266  			stream: &testutils.FakeGetFlowsServer{
   267  				FakeGRPCServerStream: fss,
   268  				OnSend: func(resp *observerpb.GetFlowsResponse) error {
   269  					if resp == nil {
   270  						return nil
   271  					}
   272  					switch resp.GetResponseTypes().(type) {
   273  					case *observerpb.GetFlowsResponse_Flow:
   274  						got.numFlows++
   275  						got.flows[resp.GetNodeName()] = append(got.flows[resp.GetNodeName()], resp.GetFlow())
   276  					case *observerpb.GetFlowsResponse_NodeStatus:
   277  						got.statusEvents = append(got.statusEvents, resp.GetNodeStatus())
   278  					}
   279  					if got.numFlows == 2 && len(got.statusEvents) == 2 {
   280  						close(done)
   281  						return io.EOF
   282  					}
   283  					return nil
   284  				},
   285  			},
   286  			want: want{
   287  				flows: map[string][]*flowpb.Flow{
   288  					"one": {&flowpb.Flow{NodeName: "one"}, &flowpb.Flow{NodeName: "one"}},
   289  				},
   290  				statusEvents: []*relaypb.NodeStatusEvent{
   291  					{
   292  						StateChange: relaypb.NodeState_NODE_CONNECTED,
   293  						NodeNames:   []string{"one"},
   294  					}, {
   295  						StateChange: relaypb.NodeState_NODE_UNAVAILABLE,
   296  						NodeNames:   []string{"two"},
   297  					},
   298  				},
   299  				err: io.EOF,
   300  				log: []string{
   301  					`level=info msg="No connection to peer two, skipping" address="192.0.2.2:4244"`,
   302  				},
   303  			},
   304  		},
   305  	}
   306  	for _, tt := range tests {
   307  		t.Run(tt.name, func(t *testing.T) {
   308  			got = &results{
   309  				flows: make(map[string][]*flowpb.Flow),
   310  			}
   311  			done = make(chan struct{})
   312  			var buf bytes.Buffer
   313  			formatter := &logrus.TextFormatter{
   314  				DisableColors:    true,
   315  				DisableTimestamp: true,
   316  			}
   317  			logger := logrus.New()
   318  			logger.SetOutput(&buf)
   319  			logger.SetFormatter(formatter)
   320  			logger.SetLevel(logrus.DebugLevel)
   321  
   322  			srv, err := NewServer(
   323  				tt.plr,
   324  				WithLogger(logger),
   325  				withObserverClientBuilder(tt.ocb),
   326  			)
   327  			assert.NoError(t, err)
   328  			err = srv.GetFlows(tt.req, tt.stream)
   329  			<-done
   330  			assert.Equal(t, tt.want.err, err)
   331  			if diff := cmp.Diff(tt.want.flows, got.flows, cmpopts.IgnoreUnexported(flowpb.Flow{})); diff != "" {
   332  				t.Errorf("Flows mismatch (-want +got):\n%s", diff)
   333  			}
   334  			if diff := cmp.Diff(tt.want.statusEvents, got.statusEvents, cmpopts.IgnoreUnexported(relaypb.NodeStatusEvent{})); diff != "" {
   335  				t.Errorf("StatusEvents mismatch (-want +got):\n%s", diff)
   336  			}
   337  			out := buf.String()
   338  			for _, msg := range tt.want.log {
   339  				assert.Contains(t, out, msg)
   340  			}
   341  		})
   342  	}
   343  }
   344  
   345  // test that Relay pick up a joining Hubble peer.
   346  func TestGetFlows_follow(t *testing.T) {
   347  	plChan := make(chan []poolTypes.Peer, 1)
   348  	pl := &testutils.FakePeerLister{
   349  		OnList: func() []poolTypes.Peer {
   350  			return <-plChan
   351  		},
   352  	}
   353  	type resp struct {
   354  		resp *observerpb.GetFlowsResponse
   355  		err  error
   356  	}
   357  	oneChan := make(chan resp, 1)
   358  	one := poolTypes.Peer{
   359  		Peer: peerTypes.Peer{
   360  			Name: "one",
   361  			Address: &net.TCPAddr{
   362  				IP:   net.ParseIP("192.0.2.1"),
   363  				Port: defaults.ServerPort,
   364  			},
   365  		},
   366  		Conn: &testutils.FakeClientConn{
   367  			OnGetState: func() connectivity.State {
   368  				return connectivity.Ready
   369  			},
   370  		},
   371  	}
   372  	twoChan := make(chan resp, 1)
   373  	two := poolTypes.Peer{
   374  		Peer: peerTypes.Peer{
   375  			Name: "two",
   376  			Address: &net.TCPAddr{
   377  				IP:   net.ParseIP("192.0.2.2"),
   378  				Port: defaults.ServerPort,
   379  			},
   380  		},
   381  		Conn: &testutils.FakeClientConn{
   382  			OnGetState: func() connectivity.State {
   383  				return connectivity.Ready
   384  			},
   385  		},
   386  	}
   387  
   388  	ocb := fakeObserverClientBuilder{
   389  		onObserverClient: func(peer *poolTypes.Peer) observerpb.ObserverClient {
   390  			return &testutils.FakeObserverClient{
   391  				OnGetFlows: func(_ context.Context, in *observerpb.GetFlowsRequest, _ ...grpc.CallOption) (observerpb.Observer_GetFlowsClient, error) {
   392  					return &testutils.FakeGetFlowsClient{
   393  						OnRecv: func() (*observerpb.GetFlowsResponse, error) {
   394  							switch peer.Name {
   395  							case "one":
   396  								r := <-oneChan
   397  								return r.resp, r.err
   398  							case "two":
   399  								r := <-twoChan
   400  								return r.resp, r.err
   401  							}
   402  							return nil, fmt.Errorf("unexpected peer %q", peer.Name)
   403  						},
   404  					}, nil
   405  				},
   406  			}
   407  		},
   408  	}
   409  	fss := &testutils.FakeGRPCServerStream{
   410  		OnContext: context.TODO,
   411  	}
   412  	seenOneFlows := atomic.Int64{}
   413  	seenTwoFlows := atomic.Int64{}
   414  	stream := &testutils.FakeGetFlowsServer{
   415  		FakeGRPCServerStream: fss,
   416  		OnSend: func(resp *observerpb.GetFlowsResponse) error {
   417  			if resp == nil {
   418  				return nil
   419  			}
   420  			switch resp.GetResponseTypes().(type) {
   421  			case *observerpb.GetFlowsResponse_Flow:
   422  				switch resp.NodeName {
   423  				case "one":
   424  					seenOneFlows.Add(1)
   425  				case "two":
   426  					seenTwoFlows.Add(1)
   427  				}
   428  			case *observerpb.GetFlowsResponse_NodeStatus:
   429  			}
   430  			return nil
   431  		},
   432  	}
   433  	srv, err := NewServer(
   434  		pl,
   435  		withObserverClientBuilder(ocb),
   436  	)
   437  	srv.opts.peerUpdateInterval = 10 * time.Millisecond
   438  	require.NoError(t, err)
   439  
   440  	plChan <- []poolTypes.Peer{one}
   441  	oneChan <- resp{
   442  		resp: &observerpb.GetFlowsResponse{
   443  			NodeName: "one",
   444  			ResponseTypes: &observerpb.GetFlowsResponse_Flow{
   445  				Flow: &flowpb.Flow{
   446  					NodeName: "one",
   447  				},
   448  			},
   449  		},
   450  	}
   451  	go func() {
   452  		err = srv.GetFlows(&observerpb.GetFlowsRequest{Follow: true}, stream)
   453  		assert.NoError(t, err)
   454  	}()
   455  	assert.Eventually(t, func() bool {
   456  		return seenOneFlows.Load() == 1
   457  	}, 10*time.Second, 10*time.Millisecond)
   458  
   459  	plChan <- []poolTypes.Peer{one, two}
   460  	oneChan <- resp{
   461  		resp: &observerpb.GetFlowsResponse{
   462  			NodeName: "one",
   463  			ResponseTypes: &observerpb.GetFlowsResponse_Flow{
   464  				Flow: &flowpb.Flow{
   465  					NodeName: "one",
   466  				},
   467  			},
   468  		},
   469  	}
   470  	twoChan <- resp{
   471  		resp: &observerpb.GetFlowsResponse{
   472  			NodeName: "two",
   473  			ResponseTypes: &observerpb.GetFlowsResponse_Flow{
   474  				Flow: &flowpb.Flow{
   475  					NodeName: "two",
   476  				},
   477  			},
   478  		},
   479  	}
   480  	assert.Eventually(t, func() bool {
   481  		return seenOneFlows.Load() == 2 && seenTwoFlows.Load() == 1
   482  	}, 10*time.Second, 10*time.Millisecond)
   483  
   484  }
   485  
   486  func TestGetNodes(t *testing.T) {
   487  	type want struct {
   488  		resp *observerpb.GetNodesResponse
   489  		err  error
   490  		log  []string
   491  	}
   492  	tests := []struct {
   493  		name string
   494  		plr  PeerLister
   495  		ocb  observerClientBuilder
   496  		req  *observerpb.GetNodesRequest
   497  		want want
   498  	}{
   499  		{
   500  			name: "1 peer without address",
   501  			plr: &testutils.FakePeerLister{
   502  				OnList: func() []poolTypes.Peer {
   503  					return []poolTypes.Peer{
   504  						{
   505  							Peer: peerTypes.Peer{
   506  								Name:    "noip",
   507  								Address: nil,
   508  							},
   509  							Conn: nil,
   510  						},
   511  					}
   512  				},
   513  			},
   514  			ocb: fakeObserverClientBuilder{},
   515  			want: want{
   516  				resp: &observerpb.GetNodesResponse{
   517  					Nodes: []*observerpb.Node{
   518  						{
   519  							Name:    "noip",
   520  							Version: "",
   521  							Address: "",
   522  							State:   relaypb.NodeState_NODE_UNAVAILABLE,
   523  							Tls: &observerpb.TLS{
   524  								Enabled:    false,
   525  								ServerName: "",
   526  							},
   527  						},
   528  					},
   529  				},
   530  				log: []string{
   531  					`level=info msg="No connection to peer noip, skipping" address="<nil>"`,
   532  				},
   533  			},
   534  		}, {
   535  			name: "2 connected peers",
   536  			plr: &testutils.FakePeerLister{
   537  				OnList: func() []poolTypes.Peer {
   538  					return []poolTypes.Peer{
   539  						{
   540  							Peer: peerTypes.Peer{
   541  								Name: "one",
   542  								Address: &net.TCPAddr{
   543  									IP:   net.ParseIP("192.0.2.1"),
   544  									Port: defaults.ServerPort,
   545  								},
   546  							},
   547  							Conn: &testutils.FakeClientConn{
   548  								OnGetState: func() connectivity.State {
   549  									return connectivity.Ready
   550  								},
   551  							},
   552  						}, {
   553  							Peer: peerTypes.Peer{
   554  								Name: "two",
   555  								Address: &net.TCPAddr{
   556  									IP:   net.ParseIP("192.0.2.2"),
   557  									Port: defaults.ServerPort,
   558  								},
   559  							},
   560  							Conn: &testutils.FakeClientConn{
   561  								OnGetState: func() connectivity.State {
   562  									return connectivity.Ready
   563  								},
   564  							},
   565  						},
   566  					}
   567  				},
   568  			},
   569  			ocb: fakeObserverClientBuilder{
   570  				onObserverClient: func(p *poolTypes.Peer) observerpb.ObserverClient {
   571  					return &testutils.FakeObserverClient{
   572  						OnServerStatus: func(_ context.Context, in *observerpb.ServerStatusRequest, _ ...grpc.CallOption) (*observerpb.ServerStatusResponse, error) {
   573  							switch p.Name {
   574  							case "one":
   575  								return &observerpb.ServerStatusResponse{
   576  									UptimeNs:  123456,
   577  									Version:   "cilium v1.9.0",
   578  									MaxFlows:  4095,
   579  									NumFlows:  4095,
   580  									SeenFlows: 11000,
   581  								}, nil
   582  							case "two":
   583  								return &observerpb.ServerStatusResponse{
   584  									UptimeNs:  555555,
   585  									Version:   "cilium v1.9.0",
   586  									MaxFlows:  2047,
   587  									NumFlows:  2020,
   588  									SeenFlows: 12000,
   589  								}, nil
   590  							default:
   591  								return nil, io.EOF
   592  							}
   593  						},
   594  					}
   595  				},
   596  			},
   597  			want: want{
   598  				resp: &observerpb.GetNodesResponse{
   599  					Nodes: []*observerpb.Node{
   600  						{
   601  							Name:      "one",
   602  							Version:   "cilium v1.9.0",
   603  							Address:   "192.0.2.1:4244",
   604  							State:     relaypb.NodeState_NODE_CONNECTED,
   605  							UptimeNs:  123456,
   606  							MaxFlows:  4095,
   607  							NumFlows:  4095,
   608  							SeenFlows: 11000,
   609  							Tls: &observerpb.TLS{
   610  								Enabled:    false,
   611  								ServerName: "",
   612  							},
   613  						}, {
   614  							Name:      "two",
   615  							Version:   "cilium v1.9.0",
   616  							Address:   "192.0.2.2:4244",
   617  							State:     relaypb.NodeState_NODE_CONNECTED,
   618  							UptimeNs:  555555,
   619  							MaxFlows:  2047,
   620  							NumFlows:  2020,
   621  							SeenFlows: 12000,
   622  							Tls: &observerpb.TLS{
   623  								Enabled:    false,
   624  								ServerName: "",
   625  							},
   626  						},
   627  					},
   628  				},
   629  			},
   630  		}, {
   631  			name: "2 connected peers with TLS",
   632  			plr: &testutils.FakePeerLister{
   633  				OnList: func() []poolTypes.Peer {
   634  					return []poolTypes.Peer{
   635  						{
   636  							Peer: peerTypes.Peer{
   637  								Name: "one",
   638  								Address: &net.TCPAddr{
   639  									IP:   net.ParseIP("192.0.2.1"),
   640  									Port: defaults.ServerPort,
   641  								},
   642  								TLSEnabled:    true,
   643  								TLSServerName: "one.default.hubble-grpc.cilium.io",
   644  							},
   645  							Conn: &testutils.FakeClientConn{
   646  								OnGetState: func() connectivity.State {
   647  									return connectivity.Ready
   648  								},
   649  							},
   650  						}, {
   651  							Peer: peerTypes.Peer{
   652  								Name: "two",
   653  								Address: &net.TCPAddr{
   654  									IP:   net.ParseIP("192.0.2.2"),
   655  									Port: defaults.ServerPort,
   656  								},
   657  								TLSEnabled:    true,
   658  								TLSServerName: "two.default.hubble-grpc.cilium.io",
   659  							},
   660  							Conn: &testutils.FakeClientConn{
   661  								OnGetState: func() connectivity.State {
   662  									return connectivity.Ready
   663  								},
   664  							},
   665  						},
   666  					}
   667  				},
   668  			},
   669  			ocb: fakeObserverClientBuilder{
   670  				onObserverClient: func(p *poolTypes.Peer) observerpb.ObserverClient {
   671  					return &testutils.FakeObserverClient{
   672  						OnServerStatus: func(_ context.Context, in *observerpb.ServerStatusRequest, _ ...grpc.CallOption) (*observerpb.ServerStatusResponse, error) {
   673  							switch p.Name {
   674  							case "one":
   675  								return &observerpb.ServerStatusResponse{
   676  									UptimeNs:  123456,
   677  									Version:   "cilium v1.9.0",
   678  									MaxFlows:  4095,
   679  									NumFlows:  4095,
   680  									SeenFlows: 11000,
   681  								}, nil
   682  							case "two":
   683  								return &observerpb.ServerStatusResponse{
   684  									UptimeNs:  555555,
   685  									Version:   "cilium v1.9.0",
   686  									MaxFlows:  2047,
   687  									NumFlows:  2020,
   688  									SeenFlows: 12000,
   689  								}, nil
   690  							default:
   691  								return nil, io.EOF
   692  							}
   693  						},
   694  					}
   695  				},
   696  			},
   697  			want: want{
   698  				resp: &observerpb.GetNodesResponse{
   699  					Nodes: []*observerpb.Node{
   700  						{
   701  							Name:      "one",
   702  							Version:   "cilium v1.9.0",
   703  							Address:   "192.0.2.1:4244",
   704  							State:     relaypb.NodeState_NODE_CONNECTED,
   705  							UptimeNs:  123456,
   706  							MaxFlows:  4095,
   707  							NumFlows:  4095,
   708  							SeenFlows: 11000,
   709  							Tls: &observerpb.TLS{
   710  								Enabled:    true,
   711  								ServerName: "one.default.hubble-grpc.cilium.io",
   712  							},
   713  						}, {
   714  							Name:      "two",
   715  							Version:   "cilium v1.9.0",
   716  							Address:   "192.0.2.2:4244",
   717  							State:     relaypb.NodeState_NODE_CONNECTED,
   718  							UptimeNs:  555555,
   719  							MaxFlows:  2047,
   720  							NumFlows:  2020,
   721  							SeenFlows: 12000,
   722  							Tls: &observerpb.TLS{
   723  								Enabled:    true,
   724  								ServerName: "two.default.hubble-grpc.cilium.io",
   725  							},
   726  						},
   727  					},
   728  				},
   729  			},
   730  		}, {
   731  			name: "1 connected peer, 1 unreachable peer",
   732  			plr: &testutils.FakePeerLister{
   733  				OnList: func() []poolTypes.Peer {
   734  					return []poolTypes.Peer{
   735  						{
   736  							Peer: peerTypes.Peer{
   737  								Name: "one",
   738  								Address: &net.TCPAddr{
   739  									IP:   net.ParseIP("192.0.2.1"),
   740  									Port: defaults.ServerPort,
   741  								},
   742  							},
   743  							Conn: &testutils.FakeClientConn{
   744  								OnGetState: func() connectivity.State {
   745  									return connectivity.Ready
   746  								},
   747  							},
   748  						}, {
   749  							Peer: peerTypes.Peer{
   750  								Name: "two",
   751  								Address: &net.TCPAddr{
   752  									IP:   net.ParseIP("192.0.2.2"),
   753  									Port: defaults.ServerPort,
   754  								},
   755  							},
   756  							Conn: &testutils.FakeClientConn{
   757  								OnGetState: func() connectivity.State {
   758  									return connectivity.TransientFailure
   759  								},
   760  							},
   761  						},
   762  					}
   763  				},
   764  			},
   765  			ocb: fakeObserverClientBuilder{
   766  				onObserverClient: func(p *poolTypes.Peer) observerpb.ObserverClient {
   767  					return &testutils.FakeObserverClient{
   768  						OnServerStatus: func(_ context.Context, in *observerpb.ServerStatusRequest, _ ...grpc.CallOption) (*observerpb.ServerStatusResponse, error) {
   769  							switch p.Name {
   770  							case "one":
   771  								return &observerpb.ServerStatusResponse{
   772  									UptimeNs:  123456,
   773  									Version:   "cilium v1.9.0",
   774  									MaxFlows:  4095,
   775  									NumFlows:  4095,
   776  									SeenFlows: 11000,
   777  								}, nil
   778  							default:
   779  								return nil, io.EOF
   780  							}
   781  						},
   782  					}
   783  				},
   784  			},
   785  			want: want{
   786  				resp: &observerpb.GetNodesResponse{
   787  					Nodes: []*observerpb.Node{
   788  						{
   789  							Name:      "one",
   790  							Version:   "cilium v1.9.0",
   791  							Address:   "192.0.2.1:4244",
   792  							State:     relaypb.NodeState_NODE_CONNECTED,
   793  							UptimeNs:  123456,
   794  							MaxFlows:  4095,
   795  							NumFlows:  4095,
   796  							SeenFlows: 11000,
   797  							Tls: &observerpb.TLS{
   798  								Enabled:    false,
   799  								ServerName: "",
   800  							},
   801  						}, {
   802  							Name:     "two",
   803  							Version:  "",
   804  							Address:  "192.0.2.2:4244",
   805  							State:    relaypb.NodeState_NODE_UNAVAILABLE,
   806  							UptimeNs: 0,
   807  							Tls: &observerpb.TLS{
   808  								Enabled:    false,
   809  								ServerName: "",
   810  							},
   811  						},
   812  					},
   813  				},
   814  				log: []string{
   815  					`level=info msg="No connection to peer two, skipping" address="192.0.2.2:4244"`,
   816  				},
   817  			},
   818  		}, {
   819  			name: "1 connected peer, 1 unreachable peer, 1 peer with error",
   820  			plr: &testutils.FakePeerLister{
   821  				OnList: func() []poolTypes.Peer {
   822  					return []poolTypes.Peer{
   823  						{
   824  							Peer: peerTypes.Peer{
   825  								Name: "one",
   826  								Address: &net.TCPAddr{
   827  									IP:   net.ParseIP("192.0.2.1"),
   828  									Port: defaults.ServerPort,
   829  								},
   830  							},
   831  							Conn: &testutils.FakeClientConn{
   832  								OnGetState: func() connectivity.State {
   833  									return connectivity.Ready
   834  								},
   835  							},
   836  						}, {
   837  							Peer: peerTypes.Peer{
   838  								Name: "two",
   839  								Address: &net.TCPAddr{
   840  									IP:   net.ParseIP("192.0.2.2"),
   841  									Port: defaults.ServerPort,
   842  								},
   843  							},
   844  							Conn: &testutils.FakeClientConn{
   845  								OnGetState: func() connectivity.State {
   846  									return connectivity.TransientFailure
   847  								},
   848  							},
   849  						}, {
   850  							Peer: peerTypes.Peer{
   851  								Name: "three",
   852  								Address: &net.TCPAddr{
   853  									IP:   net.ParseIP("192.0.2.3"),
   854  									Port: defaults.ServerPort,
   855  								},
   856  							},
   857  							Conn: &testutils.FakeClientConn{
   858  								OnGetState: func() connectivity.State {
   859  									return connectivity.Ready
   860  								},
   861  							},
   862  						},
   863  					}
   864  				},
   865  			},
   866  			ocb: fakeObserverClientBuilder{
   867  				onObserverClient: func(p *poolTypes.Peer) observerpb.ObserverClient {
   868  					return &testutils.FakeObserverClient{
   869  						OnServerStatus: func(_ context.Context, in *observerpb.ServerStatusRequest, _ ...grpc.CallOption) (*observerpb.ServerStatusResponse, error) {
   870  							switch p.Name {
   871  							case "one":
   872  								return &observerpb.ServerStatusResponse{
   873  									UptimeNs:  123456,
   874  									Version:   "cilium v1.9.0",
   875  									MaxFlows:  4095,
   876  									NumFlows:  4095,
   877  									SeenFlows: 11000,
   878  								}, nil
   879  							case "three":
   880  								return nil, status.Errorf(codes.Unimplemented, "ServerStatus not implemented")
   881  							default:
   882  								return nil, io.EOF
   883  							}
   884  						},
   885  					}
   886  				},
   887  			},
   888  			want: want{
   889  				resp: &observerpb.GetNodesResponse{
   890  					Nodes: []*observerpb.Node{
   891  						{
   892  							Name:      "one",
   893  							Version:   "cilium v1.9.0",
   894  							Address:   "192.0.2.1:4244",
   895  							UptimeNs:  123456,
   896  							MaxFlows:  4095,
   897  							NumFlows:  4095,
   898  							SeenFlows: 11000,
   899  							State:     relaypb.NodeState_NODE_CONNECTED,
   900  							Tls: &observerpb.TLS{
   901  								Enabled:    false,
   902  								ServerName: "",
   903  							},
   904  						}, {
   905  							Name:    "two",
   906  							Version: "",
   907  							Address: "192.0.2.2:4244",
   908  							State:   relaypb.NodeState_NODE_UNAVAILABLE,
   909  							Tls: &observerpb.TLS{
   910  								Enabled:    false,
   911  								ServerName: "",
   912  							},
   913  						}, {
   914  							Name:    "three",
   915  							Version: "",
   916  							Address: "192.0.2.3:4244",
   917  							State:   relaypb.NodeState_NODE_ERROR,
   918  							Tls: &observerpb.TLS{
   919  								Enabled:    false,
   920  								ServerName: "",
   921  							},
   922  						},
   923  					},
   924  				},
   925  				log: []string{
   926  					`level=info msg="No connection to peer two, skipping" address="192.0.2.2:4244"`,
   927  					`level=warning msg="Failed to retrieve server status" error="rpc error: code = Unimplemented desc = ServerStatus not implemented" peer=three`,
   928  				},
   929  			},
   930  		},
   931  	}
   932  
   933  	for _, tt := range tests {
   934  		t.Run(tt.name, func(t *testing.T) {
   935  			var buf bytes.Buffer
   936  			formatter := &logrus.TextFormatter{
   937  				DisableColors:    true,
   938  				DisableTimestamp: true,
   939  			}
   940  			logger := logrus.New()
   941  			logger.SetOutput(&buf)
   942  			logger.SetFormatter(formatter)
   943  			logger.SetLevel(logrus.DebugLevel)
   944  
   945  			srv, err := NewServer(
   946  				tt.plr,
   947  				WithLogger(logger),
   948  				withObserverClientBuilder(tt.ocb),
   949  			)
   950  			assert.NoError(t, err)
   951  			got, err := srv.GetNodes(context.Background(), tt.req)
   952  			assert.Equal(t, tt.want.err, err)
   953  			assert.Equal(t, tt.want.resp, got)
   954  			out := buf.String()
   955  			for _, msg := range tt.want.log {
   956  				assert.Contains(t, out, msg)
   957  			}
   958  		})
   959  	}
   960  }
   961  
   962  func TestGetNamespaces(t *testing.T) {
   963  	type want struct {
   964  		resp *observerpb.GetNamespacesResponse
   965  		err  error
   966  		log  []string
   967  	}
   968  	tests := []struct {
   969  		name string
   970  		plr  PeerLister
   971  		ocb  observerClientBuilder
   972  		req  *observerpb.GetNamespacesRequest
   973  		want want
   974  	}{
   975  		{
   976  			name: "get no namespaces from 1 peer without address",
   977  			plr: &testutils.FakePeerLister{
   978  				OnList: func() []poolTypes.Peer {
   979  					return []poolTypes.Peer{
   980  						{
   981  							Peer: peerTypes.Peer{
   982  								Name:    "noip",
   983  								Address: nil,
   984  							},
   985  							Conn: nil,
   986  						},
   987  					}
   988  				},
   989  			},
   990  			ocb: fakeObserverClientBuilder{
   991  				onObserverClient: func(p *poolTypes.Peer) observerpb.ObserverClient {
   992  					return &testutils.FakeObserverClient{
   993  						OnGetNamespaces: func(_ context.Context, in *observerpb.GetNamespacesRequest, _ ...grpc.CallOption) (*observerpb.GetNamespacesResponse, error) {
   994  							return nil, io.EOF
   995  						},
   996  					}
   997  				},
   998  			},
   999  			want: want{
  1000  				resp: &observerpb.GetNamespacesResponse{
  1001  					Namespaces: []*observerpb.Namespace{},
  1002  				},
  1003  				log: []string{
  1004  					`level=info msg="No connection to peer noip, skipping" address="<nil>"`,
  1005  				},
  1006  			},
  1007  		},
  1008  		{
  1009  			name: "2 connected peer, 1 unreachable peer",
  1010  			plr: &testutils.FakePeerLister{
  1011  				OnList: func() []poolTypes.Peer {
  1012  					return []poolTypes.Peer{
  1013  						{
  1014  							Peer: peerTypes.Peer{
  1015  								Name: "one",
  1016  								Address: &net.TCPAddr{
  1017  									IP:   net.ParseIP("192.0.2.1"),
  1018  									Port: defaults.ServerPort,
  1019  								},
  1020  							},
  1021  							Conn: &testutils.FakeClientConn{
  1022  								OnGetState: func() connectivity.State {
  1023  									return connectivity.Ready
  1024  								},
  1025  							},
  1026  						},
  1027  						{
  1028  							Peer: peerTypes.Peer{
  1029  								Name: "two",
  1030  								Address: &net.TCPAddr{
  1031  									IP:   net.ParseIP("192.0.2.2"),
  1032  									Port: defaults.ServerPort,
  1033  								},
  1034  							},
  1035  							Conn: &testutils.FakeClientConn{
  1036  								OnGetState: func() connectivity.State {
  1037  									return connectivity.TransientFailure
  1038  								},
  1039  							},
  1040  						},
  1041  						{
  1042  							Peer: peerTypes.Peer{
  1043  								Name: "three",
  1044  								Address: &net.TCPAddr{
  1045  									IP:   net.ParseIP("192.0.2.3"),
  1046  									Port: defaults.ServerPort,
  1047  								},
  1048  							},
  1049  							Conn: &testutils.FakeClientConn{
  1050  								OnGetState: func() connectivity.State {
  1051  									return connectivity.Ready
  1052  								},
  1053  							},
  1054  						},
  1055  					}
  1056  				},
  1057  			},
  1058  			ocb: fakeObserverClientBuilder{
  1059  				onObserverClient: func(p *poolTypes.Peer) observerpb.ObserverClient {
  1060  					return &testutils.FakeObserverClient{
  1061  						OnGetNamespaces: func(_ context.Context, in *observerpb.GetNamespacesRequest, _ ...grpc.CallOption) (*observerpb.GetNamespacesResponse, error) {
  1062  							switch p.Name {
  1063  							case "one":
  1064  								return &observerpb.GetNamespacesResponse{
  1065  									Namespaces: []*observerpb.Namespace{
  1066  										{
  1067  											Namespace: "zzz",
  1068  											Cluster:   "some-cluster",
  1069  										},
  1070  										{
  1071  											Namespace: "aaa",
  1072  											Cluster:   "some-cluster",
  1073  										},
  1074  										{
  1075  											Namespace: "bbb",
  1076  											Cluster:   "some-cluster",
  1077  										},
  1078  									},
  1079  								}, nil
  1080  							case "three":
  1081  								return &observerpb.GetNamespacesResponse{
  1082  									Namespaces: []*observerpb.Namespace{
  1083  										{
  1084  											Namespace: "zzz",
  1085  											Cluster:   "some-cluster",
  1086  										},
  1087  										{
  1088  											Namespace: "ccc",
  1089  											Cluster:   "some-cluster",
  1090  										},
  1091  										{
  1092  											Namespace: "ddd",
  1093  											Cluster:   "some-cluster",
  1094  										},
  1095  									},
  1096  								}, nil
  1097  							default:
  1098  								return nil, io.EOF
  1099  							}
  1100  						},
  1101  					}
  1102  				},
  1103  			},
  1104  			want: want{
  1105  				resp: &observerpb.GetNamespacesResponse{
  1106  					Namespaces: []*observerpb.Namespace{
  1107  						{
  1108  							Namespace: "aaa",
  1109  							Cluster:   "some-cluster",
  1110  						},
  1111  						{
  1112  							Namespace: "bbb",
  1113  							Cluster:   "some-cluster",
  1114  						},
  1115  						{
  1116  							Namespace: "ccc",
  1117  							Cluster:   "some-cluster",
  1118  						},
  1119  						{
  1120  							Namespace: "ddd",
  1121  							Cluster:   "some-cluster",
  1122  						},
  1123  						{
  1124  							Namespace: "zzz",
  1125  							Cluster:   "some-cluster",
  1126  						},
  1127  					},
  1128  				},
  1129  				log: []string{
  1130  					`level=info msg="No connection to peer two, skipping" address="192.0.2.2:4244"`,
  1131  				},
  1132  			},
  1133  		},
  1134  	}
  1135  
  1136  	for _, tt := range tests {
  1137  		t.Run(tt.name, func(t *testing.T) {
  1138  			var buf bytes.Buffer
  1139  			formatter := &logrus.TextFormatter{
  1140  				DisableColors:    true,
  1141  				DisableTimestamp: true,
  1142  			}
  1143  			logger := logrus.New()
  1144  			logger.SetOutput(&buf)
  1145  			logger.SetFormatter(formatter)
  1146  			logger.SetLevel(logrus.DebugLevel)
  1147  
  1148  			srv, err := NewServer(
  1149  				tt.plr,
  1150  				WithLogger(logger),
  1151  				withObserverClientBuilder(tt.ocb),
  1152  			)
  1153  			assert.NoError(t, err)
  1154  			got, err := srv.GetNamespaces(context.Background(), tt.req)
  1155  			assert.Equal(t, tt.want.err, err)
  1156  			assert.Equal(t, tt.want.resp, got)
  1157  			out := buf.String()
  1158  			for _, msg := range tt.want.log {
  1159  				assert.Contains(t, out, msg)
  1160  			}
  1161  		})
  1162  	}
  1163  }
  1164  
  1165  func TestServerStatus(t *testing.T) {
  1166  	type want struct {
  1167  		resp *observerpb.ServerStatusResponse
  1168  		err  error
  1169  		log  []string
  1170  	}
  1171  	tests := []struct {
  1172  		name string
  1173  		plr  PeerLister
  1174  		ocb  observerClientBuilder
  1175  		req  *observerpb.ServerStatusRequest
  1176  		want want
  1177  	}{
  1178  		{
  1179  			name: "1 peer without address",
  1180  			plr: &testutils.FakePeerLister{
  1181  				OnList: func() []poolTypes.Peer {
  1182  					return []poolTypes.Peer{
  1183  						{
  1184  							Peer: peerTypes.Peer{
  1185  								Name:    "noip",
  1186  								Address: nil,
  1187  							},
  1188  							Conn: nil,
  1189  						},
  1190  					}
  1191  				},
  1192  			},
  1193  			ocb: fakeObserverClientBuilder{},
  1194  			want: want{
  1195  				resp: &observerpb.ServerStatusResponse{
  1196  					Version:             "hubble-relay",
  1197  					NumFlows:            0,
  1198  					MaxFlows:            0,
  1199  					SeenFlows:           0,
  1200  					UptimeNs:            0,
  1201  					FlowsRate:           0,
  1202  					NumConnectedNodes:   &wrapperspb.UInt32Value{Value: 0},
  1203  					NumUnavailableNodes: &wrapperspb.UInt32Value{Value: 1},
  1204  					UnavailableNodes:    []string{"noip"},
  1205  				},
  1206  				log: []string{
  1207  					`level=info msg="No connection to peer noip, skipping" address="<nil>"`,
  1208  				},
  1209  			},
  1210  		}, {
  1211  			name: "2 connected peers",
  1212  			plr: &testutils.FakePeerLister{
  1213  				OnList: func() []poolTypes.Peer {
  1214  					return []poolTypes.Peer{
  1215  						{
  1216  							Peer: peerTypes.Peer{
  1217  								Name: "one",
  1218  								Address: &net.TCPAddr{
  1219  									IP:   net.ParseIP("192.0.2.1"),
  1220  									Port: defaults.ServerPort,
  1221  								},
  1222  							},
  1223  							Conn: &testutils.FakeClientConn{
  1224  								OnGetState: func() connectivity.State {
  1225  									return connectivity.Ready
  1226  								},
  1227  							},
  1228  						}, {
  1229  							Peer: peerTypes.Peer{
  1230  								Name: "two",
  1231  								Address: &net.TCPAddr{
  1232  									IP:   net.ParseIP("192.0.2.2"),
  1233  									Port: defaults.ServerPort,
  1234  								},
  1235  							},
  1236  							Conn: &testutils.FakeClientConn{
  1237  								OnGetState: func() connectivity.State {
  1238  									return connectivity.Ready
  1239  								},
  1240  							},
  1241  						},
  1242  					}
  1243  				},
  1244  			},
  1245  			ocb: fakeObserverClientBuilder{
  1246  				onObserverClient: func(p *poolTypes.Peer) observerpb.ObserverClient {
  1247  					return &testutils.FakeObserverClient{
  1248  						OnServerStatus: func(_ context.Context, in *observerpb.ServerStatusRequest, _ ...grpc.CallOption) (*observerpb.ServerStatusResponse, error) {
  1249  							switch p.Name {
  1250  							case "one":
  1251  								return &observerpb.ServerStatusResponse{
  1252  									NumFlows:  1111,
  1253  									MaxFlows:  1111,
  1254  									SeenFlows: 1111,
  1255  									FlowsRate: 1,
  1256  									UptimeNs:  111111111,
  1257  								}, nil
  1258  							case "two":
  1259  								return &observerpb.ServerStatusResponse{
  1260  									NumFlows:  2222,
  1261  									MaxFlows:  2222,
  1262  									SeenFlows: 2222,
  1263  									FlowsRate: 2,
  1264  									UptimeNs:  222222222,
  1265  								}, nil
  1266  							default:
  1267  								return nil, io.EOF
  1268  							}
  1269  						},
  1270  					}
  1271  				},
  1272  			},
  1273  			want: want{
  1274  				resp: &observerpb.ServerStatusResponse{
  1275  					Version:             "hubble-relay",
  1276  					NumFlows:            3333,
  1277  					MaxFlows:            3333,
  1278  					SeenFlows:           3333,
  1279  					FlowsRate:           3,
  1280  					UptimeNs:            222222222,
  1281  					NumConnectedNodes:   &wrapperspb.UInt32Value{Value: 2},
  1282  					NumUnavailableNodes: &wrapperspb.UInt32Value{Value: 0},
  1283  				},
  1284  			},
  1285  		}, {
  1286  			name: "1 connected peer, 1 unreachable peer",
  1287  			plr: &testutils.FakePeerLister{
  1288  				OnList: func() []poolTypes.Peer {
  1289  					return []poolTypes.Peer{
  1290  						{
  1291  							Peer: peerTypes.Peer{
  1292  								Name: "one",
  1293  								Address: &net.TCPAddr{
  1294  									IP:   net.ParseIP("192.0.2.1"),
  1295  									Port: defaults.ServerPort,
  1296  								},
  1297  							},
  1298  							Conn: &testutils.FakeClientConn{
  1299  								OnGetState: func() connectivity.State {
  1300  									return connectivity.Ready
  1301  								},
  1302  							},
  1303  						}, {
  1304  							Peer: peerTypes.Peer{
  1305  								Name: "two",
  1306  								Address: &net.TCPAddr{
  1307  									IP:   net.ParseIP("192.0.2.2"),
  1308  									Port: defaults.ServerPort,
  1309  								},
  1310  							},
  1311  							Conn: &testutils.FakeClientConn{
  1312  								OnGetState: func() connectivity.State {
  1313  									return connectivity.TransientFailure
  1314  								},
  1315  							},
  1316  						},
  1317  					}
  1318  				},
  1319  			},
  1320  			ocb: fakeObserverClientBuilder{
  1321  				onObserverClient: func(p *poolTypes.Peer) observerpb.ObserverClient {
  1322  					return &testutils.FakeObserverClient{
  1323  						OnServerStatus: func(_ context.Context, in *observerpb.ServerStatusRequest, _ ...grpc.CallOption) (*observerpb.ServerStatusResponse, error) {
  1324  							switch p.Name {
  1325  							case "one":
  1326  								return &observerpb.ServerStatusResponse{
  1327  									NumFlows:  1111,
  1328  									MaxFlows:  1111,
  1329  									SeenFlows: 1111,
  1330  									FlowsRate: 1,
  1331  									UptimeNs:  111111111,
  1332  								}, nil
  1333  							default:
  1334  								return nil, io.EOF
  1335  							}
  1336  						},
  1337  					}
  1338  				},
  1339  			},
  1340  			want: want{
  1341  				resp: &observerpb.ServerStatusResponse{
  1342  					Version:             "hubble-relay",
  1343  					NumFlows:            1111,
  1344  					MaxFlows:            1111,
  1345  					SeenFlows:           1111,
  1346  					FlowsRate:           1,
  1347  					UptimeNs:            111111111,
  1348  					NumConnectedNodes:   &wrapperspb.UInt32Value{Value: 1},
  1349  					NumUnavailableNodes: &wrapperspb.UInt32Value{Value: 1},
  1350  					UnavailableNodes:    []string{"two"},
  1351  				},
  1352  				log: []string{
  1353  					`level=info msg="No connection to peer two, skipping" address="192.0.2.2:4244"`,
  1354  				},
  1355  			},
  1356  		}, {
  1357  			name: "2 unreachable peers",
  1358  			plr: &testutils.FakePeerLister{
  1359  				OnList: func() []poolTypes.Peer {
  1360  					return []poolTypes.Peer{
  1361  						{
  1362  							Peer: peerTypes.Peer{
  1363  								Name: "one",
  1364  								Address: &net.TCPAddr{
  1365  									IP:   net.ParseIP("192.0.2.1"),
  1366  									Port: defaults.ServerPort,
  1367  								},
  1368  							},
  1369  							Conn: &testutils.FakeClientConn{
  1370  								OnGetState: func() connectivity.State {
  1371  									return connectivity.TransientFailure
  1372  								},
  1373  							},
  1374  						}, {
  1375  							Peer: peerTypes.Peer{
  1376  								Name: "two",
  1377  								Address: &net.TCPAddr{
  1378  									IP:   net.ParseIP("192.0.2.2"),
  1379  									Port: defaults.ServerPort,
  1380  								},
  1381  							},
  1382  							Conn: &testutils.FakeClientConn{
  1383  								OnGetState: func() connectivity.State {
  1384  									return connectivity.TransientFailure
  1385  								},
  1386  							},
  1387  						},
  1388  					}
  1389  				},
  1390  			},
  1391  			ocb: fakeObserverClientBuilder{
  1392  				onObserverClient: func(p *poolTypes.Peer) observerpb.ObserverClient {
  1393  					return &testutils.FakeObserverClient{
  1394  						OnServerStatus: func(_ context.Context, in *observerpb.ServerStatusRequest, _ ...grpc.CallOption) (*observerpb.ServerStatusResponse, error) {
  1395  							return nil, io.EOF
  1396  						},
  1397  					}
  1398  				},
  1399  			},
  1400  			want: want{
  1401  				resp: &observerpb.ServerStatusResponse{
  1402  					Version:             "hubble-relay",
  1403  					NumFlows:            0,
  1404  					MaxFlows:            0,
  1405  					SeenFlows:           0,
  1406  					UptimeNs:            0,
  1407  					NumConnectedNodes:   &wrapperspb.UInt32Value{Value: 0},
  1408  					NumUnavailableNodes: &wrapperspb.UInt32Value{Value: 2},
  1409  					UnavailableNodes:    []string{"one", "two"},
  1410  				},
  1411  				log: []string{
  1412  					`level=info msg="No connection to peer one, skipping" address="192.0.2.1:4244"`,
  1413  					`level=info msg="No connection to peer two, skipping" address="192.0.2.2:4244"`,
  1414  				},
  1415  			},
  1416  		},
  1417  	}
  1418  
  1419  	for _, tt := range tests {
  1420  		t.Run(tt.name, func(t *testing.T) {
  1421  			var buf bytes.Buffer
  1422  			formatter := &logrus.TextFormatter{
  1423  				DisableColors:    true,
  1424  				DisableTimestamp: true,
  1425  			}
  1426  			logger := logrus.New()
  1427  			logger.SetOutput(&buf)
  1428  			logger.SetFormatter(formatter)
  1429  			logger.SetLevel(logrus.DebugLevel)
  1430  
  1431  			srv, err := NewServer(
  1432  				tt.plr,
  1433  				WithLogger(logger),
  1434  				withObserverClientBuilder(tt.ocb),
  1435  			)
  1436  			assert.NoError(t, err)
  1437  			got, err := srv.ServerStatus(context.Background(), tt.req)
  1438  			assert.Equal(t, tt.want.err, err)
  1439  			assert.Equal(t, tt.want.resp, got)
  1440  			out := buf.String()
  1441  			for _, msg := range tt.want.log {
  1442  				assert.Contains(t, out, msg)
  1443  			}
  1444  		})
  1445  	}
  1446  }
  1447  
  1448  type fakeObserverClientBuilder struct {
  1449  	onObserverClient func(*poolTypes.Peer) observerpb.ObserverClient
  1450  }
  1451  
  1452  func (b fakeObserverClientBuilder) observerClient(p *poolTypes.Peer) observerpb.ObserverClient {
  1453  	if b.onObserverClient != nil {
  1454  		return b.onObserverClient(p)
  1455  	}
  1456  	panic("OnObserverClient not set")
  1457  }