code.vegaprotocol.io/vega@v0.79.0/wallet/api/node/round_robin_selector_test.go (about) 1 // Copyright (C) 2023 Gobalsky Labs Limited 2 // 3 // This program is free software: you can redistribute it and/or modify 4 // it under the terms of the GNU Affero General Public License as 5 // published by the Free Software Foundation, either version 3 of the 6 // License, or (at your option) any later version. 7 // 8 // This program is distributed in the hope that it will be useful, 9 // but WITHOUT ANY WARRANTY; without even the implied warranty of 10 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 // GNU Affero General Public License for more details. 12 // 13 // You should have received a copy of the GNU Affero General Public License 14 // along with this program. If not, see <http://www.gnu.org/licenses/>. 15 16 package node_test 17 18 import ( 19 "context" 20 "fmt" 21 "testing" 22 23 vgrand "code.vegaprotocol.io/vega/libs/rand" 24 "code.vegaprotocol.io/vega/wallet/api/node" 25 nodemocks "code.vegaprotocol.io/vega/wallet/api/node/mocks" 26 "code.vegaprotocol.io/vega/wallet/api/node/types" 27 28 "github.com/golang/mock/gomock" 29 "github.com/stretchr/testify/assert" 30 "github.com/stretchr/testify/require" 31 ) 32 33 func TestRoundRobinSelector(t *testing.T) { 34 t.Run("Returns one of the healthiest node", testRoundRobinSelectorReturnsTheFirstHealthyNode) 35 t.Run("Stopping the selector stops all nodes", testRoundRobinSelectorStoppingTheSelectorStopsAllNodes) 36 } 37 38 func testRoundRobinSelectorReturnsTheFirstHealthyNode(t *testing.T) { 39 ctx := context.Background() 40 log := newTestLogger(t) 41 ctrl := gomock.NewController(t) 42 43 // given 44 node0 := nodemocks.NewMockNode(ctrl) 45 node0.EXPECT().Host().AnyTimes().Return("node-0") 46 47 node1 := nodemocks.NewMockNode(ctrl) 48 node1.EXPECT().Host().AnyTimes().Return("node-1") 49 50 node2 := nodemocks.NewMockNode(ctrl) 51 node2.EXPECT().Host().AnyTimes().Return("node-2") 52 53 node3 := nodemocks.NewMockNode(ctrl) 54 node3.EXPECT().Host().AnyTimes().Return("node-3") 55 56 node4 := nodemocks.NewMockNode(ctrl) 57 node4.EXPECT().Host().AnyTimes().Return("node-4") 58 59 chainID := vgrand.RandomStr(5) 60 61 latestStats := types.Statistics{ 62 BlockHash: vgrand.RandomStr(5), 63 BlockHeight: 987654321, 64 ChainID: chainID, 65 VegaTime: "123456789", 66 } 67 68 lateStats := types.Statistics{ 69 BlockHash: vgrand.RandomStr(5), 70 BlockHeight: 987654310, 71 ChainID: chainID, 72 VegaTime: "123456780", 73 } 74 75 veryLateStats := types.Statistics{ 76 BlockHash: vgrand.RandomStr(5), 77 BlockHeight: 987654300, 78 ChainID: chainID, 79 VegaTime: "123456750", 80 } 81 82 // when 83 selector, err := node.NewRoundRobinSelector(log, node0, node1, node2, node3, node4) 84 85 // then 86 require.NoError(t, err) 87 88 // given all nodes are healthy 89 node0.EXPECT().Statistics(ctx).Times(10).Return(latestStats, nil) 90 node1.EXPECT().Statistics(ctx).Times(10).Return(latestStats, nil) 91 node2.EXPECT().Statistics(ctx).Times(10).Return(latestStats, nil) 92 node3.EXPECT().Statistics(ctx).Times(10).Return(latestStats, nil) 93 node4.EXPECT().Statistics(ctx).Times(10).Return(latestStats, nil) 94 95 // This tests the round-robbin capability with healthy node only. 96 for i := 0; i < 10; i++ { 97 // when 98 selectedNode, err := selector.Node(ctx, noReporting) 99 100 // then it returns the next healthy node 101 require.NoError(t, err) 102 require.NotEmpty(t, selectedNode) 103 expectedNodeHost := fmt.Sprintf("node-%d", i%5) 104 assert.Equal(t, expectedNodeHost, selectedNode.Host(), fmt.Sprintf("expected %s, but got %s after %d iterations", expectedNodeHost, selectedNode.Host(), i)) 105 } 106 107 node0.EXPECT().Statistics(ctx).Times(1).Return(latestStats, nil) 108 node1.EXPECT().Statistics(ctx).Times(1).Return(latestStats, nil) 109 node2.EXPECT().Statistics(ctx).Times(1).Return(latestStats, nil) 110 node3.EXPECT().Statistics(ctx).Times(1).Return(latestStats, nil) 111 node4.EXPECT().Statistics(ctx).Times(1).Return(latestStats, nil) 112 113 // when 114 selectedNode, err := selector.Node(ctx, noReporting) 115 116 // then it returns the next healthy node 117 require.NoError(t, err) 118 require.NotEmpty(t, selectedNode) 119 assert.Equal(t, "node-0", selectedNode.Host()) 120 121 // given `node-1` and `node-2` become unhealthy, 122 // and the latest node selected node is the `node-0` 123 node0.EXPECT().Statistics(ctx).Times(1).Return(latestStats, nil) 124 node1.EXPECT().Statistics(ctx).Times(1).Return(lateStats, nil) 125 node2.EXPECT().Statistics(ctx).Times(1).Return(veryLateStats, nil) 126 node3.EXPECT().Statistics(ctx).Times(1).Return(latestStats, nil) 127 node4.EXPECT().Statistics(ctx).Times(1).Return(latestStats, nil) 128 129 // when 130 selectedNode, err = selector.Node(ctx, noReporting) 131 132 // then it returns the next healthy node `node-3` 133 require.NoError(t, err) 134 require.NotEmpty(t, selectedNode) 135 assert.Equal(t, "node-3", selectedNode.Host()) 136 137 // given `node-0` and `node-4` become unhealthy, 138 // and the latest node selected node is the `node-3` 139 node0.EXPECT().Statistics(ctx).Times(1).Return(lateStats, nil) 140 node1.EXPECT().Statistics(ctx).Times(1).Return(latestStats, nil) 141 node2.EXPECT().Statistics(ctx).Times(1).Return(latestStats, nil) 142 node3.EXPECT().Statistics(ctx).Times(1).Return(latestStats, nil) 143 node4.EXPECT().Statistics(ctx).Times(1).Return(lateStats, nil) 144 145 // when 146 selectedNode, err = selector.Node(ctx, noReporting) 147 148 // then it returns the next healthy node `node-1` 149 require.NoError(t, err) 150 require.NotEmpty(t, selectedNode) 151 assert.Equal(t, "node-1", selectedNode.Host()) 152 153 // given `node-0`, `node-1` and `node-2` become unhealthy, 154 // and the latest node selected node is the `node-4` 155 // This situation is special because we have as many veryLateStats as latestStats, 156 // so there is no clear source of truth. At that point, the selector has to 157 // pick a winner between the two, based on other properties, like the block 158 // height. 159 node0.EXPECT().Statistics(ctx).Times(1).Return(lateStats, nil) 160 node1.EXPECT().Statistics(ctx).Times(1).Return(veryLateStats, nil) 161 node2.EXPECT().Statistics(ctx).Times(1).Return(veryLateStats, nil) 162 node3.EXPECT().Statistics(ctx).Times(1).Return(latestStats, nil) 163 node4.EXPECT().Statistics(ctx).Times(1).Return(latestStats, nil) 164 165 // when 166 selectedNode, err = selector.Node(ctx, noReporting) 167 168 // then it returns the next healthy node `node-3` 169 require.NoError(t, err) 170 require.NotEmpty(t, selectedNode) 171 assert.Equal(t, "node-3", selectedNode.Host()) 172 173 // EDGE CASE ! Ideally we would like this to not happen, but it does because 174 // we can't do otherwise... 175 // For more details, read the comments on the algorithm implementation. 176 177 // given `node-0`, `node-2`, `node-4`, node-3 become unhealthy, 178 // and the latest node selected node is the `node-3` 179 node0.EXPECT().Statistics(ctx).Times(1).Return(veryLateStats, nil) 180 node1.EXPECT().Statistics(ctx).Times(1).Return(latestStats, nil) 181 node2.EXPECT().Statistics(ctx).Times(1).Return(veryLateStats, nil) 182 node3.EXPECT().Statistics(ctx).Times(1).Return(latestStats, nil) 183 node4.EXPECT().Statistics(ctx).Times(1).Return(veryLateStats, nil) 184 185 // when 186 selectedNode, err = selector.Node(ctx, noReporting) 187 188 // then it returns the next healthy node `node-4` 189 require.NoError(t, err) 190 require.NotEmpty(t, selectedNode) 191 assert.Equal(t, "node-4", selectedNode.Host()) 192 193 // given all nodes except one don't respond except one. 194 node0.EXPECT().Statistics(ctx).Times(1).Return(types.Statistics{}, assert.AnError) 195 node1.EXPECT().Statistics(ctx).Times(1).Return(types.Statistics{}, assert.AnError) 196 node2.EXPECT().Statistics(ctx).Times(1).Return(latestStats, nil) 197 node3.EXPECT().Statistics(ctx).Times(1).Return(types.Statistics{}, assert.AnError) 198 node4.EXPECT().Statistics(ctx).Times(1).Return(types.Statistics{}, assert.AnError) 199 200 // when 201 selectedNode, err = selector.Node(ctx, noReporting) 202 203 // then it returns the next healthy node `node-2` 204 require.NoError(t, err) 205 require.NotEmpty(t, selectedNode) 206 assert.Equal(t, "node-2", selectedNode.Host()) 207 208 // given all nodes except one don't respond except one. 209 node0.EXPECT().Statistics(ctx).Times(1).Return(types.Statistics{}, assert.AnError) 210 node1.EXPECT().Statistics(ctx).Times(1).Return(types.Statistics{}, assert.AnError) 211 node2.EXPECT().Statistics(ctx).Times(1).Return(types.Statistics{}, assert.AnError) 212 node3.EXPECT().Statistics(ctx).Times(1).Return(types.Statistics{}, assert.AnError) 213 node4.EXPECT().Statistics(ctx).Times(1).Return(types.Statistics{}, assert.AnError) 214 215 // when 216 selectedNode, err = selector.Node(ctx, noReporting) 217 218 // then 219 require.ErrorIs(t, err, node.ErrNoHealthyNodeAvailable) 220 require.Empty(t, selectedNode) 221 } 222 223 func testRoundRobinSelectorStoppingTheSelectorStopsAllNodes(t *testing.T) { 224 // given 225 log := newTestLogger(t) 226 ctrl := gomock.NewController(t) 227 228 closingHost1 := nodemocks.NewMockNode(ctrl) 229 closingHost1.EXPECT().Stop().Times(1).Return(nil) 230 231 failedClosingHost := nodemocks.NewMockNode(ctrl) 232 failedClosingHost.EXPECT().Stop().Times(1).Return(assert.AnError) 233 234 closingHost2 := nodemocks.NewMockNode(ctrl) 235 closingHost2.EXPECT().Stop().Times(1).Return(nil) 236 237 // when 238 selector, err := node.NewRoundRobinSelector(log, 239 closingHost1, 240 failedClosingHost, 241 closingHost2, 242 ) 243 244 // then 245 require.NoError(t, err) 246 247 // when 248 require.NotPanics(t, func() { 249 selector.Stop() 250 }) 251 }