github.com/hashgraph/hedera-sdk-go/v2@v2.48.0/managed_network_unit_test.go (about)

     1  //go:build all || unit
     2  // +build all unit
     3  
     4  package hedera
     5  
     6  /*-
     7   *
     8   * Hedera Go SDK
     9   *
    10   * Copyright (C) 2020 - 2024 Hedera Hashgraph, LLC
    11   *
    12   * Licensed under the Apache License, Version 2.0 (the "License");
    13   * you may not use this file except in compliance with the License.
    14   * You may obtain a copy of the License at
    15   *
    16   *      http://www.apache.org/licenses/LICENSE-2.0
    17   *
    18   * Unless required by applicable law or agreed to in writing, software
    19   * distributed under the License is distributed on an "AS IS" BASIS,
    20   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    21   * See the License for the specific language governing permissions and
    22   * limitations under the License.
    23   *
    24   */
    25  
    26  import (
    27  	"math"
    28  	"testing"
    29  	"time"
    30  
    31  	"github.com/pkg/errors"
    32  	"github.com/stretchr/testify/require"
    33  )
    34  
    35  func (this *_ManagedNetwork) removeNodeFromHealthyNodes(nodeToRemove _IManagedNode) {
    36  	newHealthyNodes := make([]_IManagedNode, 0)
    37  
    38  	for _, node := range this.healthyNodes {
    39  		if node != nodeToRemove {
    40  			newHealthyNodes = append(newHealthyNodes, node)
    41  		}
    42  	}
    43  
    44  	this.healthyNodes = newHealthyNodes
    45  }
    46  
    47  func newMockNodes() map[string]_IManagedNode {
    48  	address1, _ := _ManagedNodeAddressFromString("node1:50211")
    49  	address2, _ := _ManagedNodeAddressFromString("node1:50212")
    50  	address3, _ := _ManagedNodeAddressFromString("node2:50211")
    51  
    52  	return map[string]_IManagedNode{
    53  		"node1:50211": &mockManagedNode{address: address1, healthy: true},
    54  		"node1:50212": &mockManagedNode{address: address2, healthy: true},
    55  		"node2:50211": &mockManagedNode{address: address3, healthy: true},
    56  	}
    57  }
    58  
    59  type mockManagedNode struct {
    60  	address                    *_ManagedNodeAddress
    61  	currentBackoff             time.Duration
    62  	lastUsed                   time.Time
    63  	useCount                   int64
    64  	minBackoff                 time.Duration
    65  	maxBackoff                 time.Duration
    66  	badGrpcStatusCount         int64
    67  	readmitTime                *time.Time
    68  	healthy                    bool
    69  	minBackoffCalled           bool
    70  	maxBackoffCalled           bool
    71  	setVerifyCertificateCalled bool
    72  	toSecureCalled             bool
    73  	toInsecureCalled           bool
    74  }
    75  type mockManagedNodeWithError struct {
    76  	mockManagedNode
    77  }
    78  
    79  func (m *mockManagedNodeWithError) _Close() error {
    80  	return errors.New("closing error")
    81  }
    82  
    83  func (m *mockManagedNode) _GetAddress() string {
    84  	return m.address._String()
    85  }
    86  
    87  func (m *mockManagedNode) _GetKey() string {
    88  	return m.address._String()
    89  }
    90  
    91  func (m *mockManagedNode) _IsHealthy() bool {
    92  	return m.healthy
    93  }
    94  
    95  func (m *mockManagedNode) _Close() error {
    96  	return nil
    97  }
    98  
    99  func (m *mockManagedNode) _DecreaseBackoff() {
   100  
   101  }
   102  
   103  func (m *mockManagedNode) _IncreaseBackoff() {
   104  	m.healthy = false
   105  }
   106  
   107  func (m *mockManagedNode) _ResetBackoff() {
   108  	// No need to implement this for the test
   109  }
   110  
   111  func (m *mockManagedNode) _GetReadmitTime() *time.Time {
   112  	return m.readmitTime
   113  }
   114  
   115  func (m *mockManagedNode) _GetAttempts() int64 {
   116  	return 0
   117  }
   118  
   119  func (m *mockManagedNode) _GetLastUsed() time.Time {
   120  	return time.Now()
   121  }
   122  
   123  func (m *mockManagedNode) _GetManagedNode() *_ManagedNode {
   124  	return nil
   125  }
   126  
   127  func (m *mockManagedNode) _ToSecure() _IManagedNode {
   128  	m.toSecureCalled = true
   129  	return m
   130  }
   131  
   132  func (m *mockManagedNode) _ToInsecure() _IManagedNode {
   133  	m.toInsecureCalled = true
   134  	return m
   135  }
   136  
   137  func (m *mockManagedNode) _SetMinBackoff(minBackoff time.Duration) {
   138  	m.minBackoff = minBackoff
   139  	m.minBackoffCalled = true
   140  }
   141  
   142  func (m *mockManagedNode) _SetMaxBackoff(maxBackoff time.Duration) {
   143  	m.maxBackoff = maxBackoff
   144  	m.maxBackoffCalled = true
   145  }
   146  
   147  func (m *mockManagedNode) _GetMinBackoff() time.Duration {
   148  	return time.Duration(0)
   149  }
   150  
   151  func (m *mockManagedNode) _GetMaxBackoff() time.Duration {
   152  	return time.Duration(0)
   153  }
   154  
   155  func (m *mockManagedNode) _Wait() time.Duration {
   156  	return time.Duration(0)
   157  }
   158  
   159  func (m *mockManagedNode) _GetUseCount() int64 {
   160  	return 0
   161  }
   162  
   163  func (m *mockManagedNode) _SetVerifyCertificate(verify bool) {
   164  	m.setVerifyCertificateCalled = true
   165  }
   166  
   167  func (m *mockManagedNode) _GetVerifyCertificate() bool {
   168  	return true
   169  }
   170  
   171  func (m *mockManagedNode) _InUse() {
   172  }
   173  
   174  func TestUnitManagedNetworkSetGet(t *testing.T) {
   175  	t.Parallel()
   176  
   177  	mn := _NewManagedNetwork()
   178  	mockNodes := newMockNodes()
   179  	err := mn._SetNetwork(mockNodes)
   180  	require.NoError(t, err)
   181  	ledgerId, err := LedgerIDFromString("mainnet")
   182  	require.NoError(t, err)
   183  	mn._SetLedgerID(*ledgerId)
   184  	mn._SetMaxNodeAttempts(10)
   185  	mn._SetMinBackoff(1 * time.Second)
   186  	mn._SetMaxBackoff(2 * time.Second)
   187  	mn._SetMaxNodesPerTransaction(3)
   188  	mn._SetTransportSecurity(false)
   189  	mn._SetVerifyCertificate(false)
   190  	mn._SetMinNodeReadmitPeriod(4 * time.Second)
   191  	mn._SetMaxNodeReadmitPeriod(5 * time.Second)
   192  
   193  	require.Equal(t, 10, mn.maxNodeAttempts)
   194  	require.Equal(t, 1*time.Second, mn._GetMinBackoff())
   195  	require.Equal(t, 2*time.Second, mn._GetMaxBackoff())
   196  	require.Equal(t, 3, *mn.maxNodesPerTransaction)
   197  	require.Equal(t, ledgerId, mn._GetLedgerID())
   198  	require.False(t, mn.transportSecurity)
   199  	require.False(t, mn._GetVerifyCertificate())
   200  	require.Equal(t, 4*time.Second, mn.minNodeReadmitPeriod)
   201  	require.Equal(t, 5*time.Second, mn.maxNodeReadmitPeriod)
   202  	for _, node := range mockNodes {
   203  		mockNode, ok := node.(*mockManagedNode)
   204  		require.True(t, ok, "node should be of type *mockManagedNode")
   205  
   206  		require.True(t, mockNode.minBackoffCalled, "minBackoffCalled should be true")
   207  		require.True(t, mockNode.maxBackoffCalled, "maxBackoffCalled should be true")
   208  		require.True(t, mockNode.setVerifyCertificateCalled, "setVerifyCertificateCalled should be true")
   209  		// Should not be called, as those are false by default
   210  		require.False(t, mockNode.toSecureCalled, "toSecureCalled should be false")
   211  		require.False(t, mockNode.toInsecureCalled, "toInsecureCalled should be false")
   212  	}
   213  	mn._SetTransportSecurity(true)
   214  	mn._SetVerifyCertificate(true)
   215  	for _, node := range mockNodes {
   216  		mockNode, ok := node.(*mockManagedNode)
   217  		require.True(t, ok, "node should be of type *mockManagedNode")
   218  
   219  		require.True(t, mockNode.setVerifyCertificateCalled, "setVerifyCertificateCalled should be true")
   220  		require.True(t, mockNode.toSecureCalled, "toSecureCalled should be true")
   221  	}
   222  	mn._SetTransportSecurity(false)
   223  	for _, node := range mockNodes {
   224  		mockNode, ok := node.(*mockManagedNode)
   225  		require.True(t, ok, "node should be of type *mockManagedNode")
   226  		require.True(t, mockNode.toInsecureCalled, "toInsecureCalled should be true")
   227  	}
   228  }
   229  
   230  func TestUnitNewManagedNetwork(t *testing.T) {
   231  	t.Parallel()
   232  
   233  	mn := _NewManagedNetwork()
   234  
   235  	require.NotNil(t, mn.network)
   236  	require.NotNil(t, mn.nodes)
   237  	require.NotNil(t, mn.healthyNodes)
   238  	require.Equal(t, -1, mn._GetMaxNodeAttempts())
   239  	require.Equal(t, 8*time.Second, mn.minBackoff)
   240  	require.Equal(t, 1*time.Hour, mn.maxBackoff)
   241  	require.Nil(t, mn.maxNodesPerTransaction)
   242  	require.Nil(t, mn.ledgerID)
   243  	require.False(t, mn.transportSecurity)
   244  	require.False(t, mn.verifyCertificate)
   245  	require.Equal(t, 8*time.Second, mn._GetMinNodeReadmitPeriod())
   246  	require.Equal(t, 1*time.Hour, mn._GetMaxNodeReadmitPeriod())
   247  }
   248  
   249  func TestUnitSetNetwork(t *testing.T) {
   250  	t.Parallel()
   251  
   252  	mn := _NewManagedNetwork()
   253  	mockNodes := newMockNodes()
   254  	err := mn._SetNetwork(mockNodes)
   255  	require.NoError(t, err)
   256  
   257  	// Check if the nodes are properly set in the _ManagedNetwork
   258  	require.Equal(t, 3, len(mn.nodes))
   259  	for _, node := range mn.nodes {
   260  		require.Contains(t, mockNodes, node._GetAddress())
   261  	}
   262  
   263  	// Check if the healthy nodes are properly set in the _ManagedNetwork
   264  	require.Equal(t, 3, len(mn.healthyNodes))
   265  	for _, node := range mn.healthyNodes {
   266  		require.Contains(t, mockNodes, node._GetAddress())
   267  	}
   268  
   269  	mockNodes["node1:50211"].(*mockManagedNode).healthy = false
   270  	err = mn._SetNetwork(mockNodes)
   271  	require.NoError(t, err)
   272  	// Check if only the healthy nodes are properly set in the _ManagedNetwork
   273  	require.Equal(t, 2, len(mn.healthyNodes))
   274  	for _, node := range mn.healthyNodes {
   275  		require.True(t, node._IsHealthy())
   276  	}
   277  
   278  	// Check if the nodes are properly set in the _ManagedNetwork
   279  	require.Equal(t, 3, len(mn.nodes))
   280  	for _, node := range mn.nodes {
   281  		require.Contains(t, mockNodes, node._GetAddress())
   282  	}
   283  }
   284  
   285  func TestUnitSetNetworkWithErorr(t *testing.T) {
   286  	t.Parallel()
   287  
   288  	mn := _NewManagedNetwork()
   289  	mockNodes := newMockNodes()
   290  	address4, _ := _ManagedNodeAddressFromString("node1:50213")
   291  	mockNodesWithError := map[string]_IManagedNode{
   292  		"node1:50213": &mockManagedNodeWithError{
   293  			mockManagedNode: mockManagedNode{address: address4, healthy: true},
   294  		},
   295  	}
   296  
   297  	err := mn._SetNetwork(mockNodesWithError)
   298  	require.NoError(t, err)
   299  	// Add a new node, should error, because existing node return an error on close
   300  	err = mn._SetNetwork(mockNodes)
   301  	require.Error(t, err)
   302  }
   303  
   304  func TestUnitManagedNetworkCloseWithError(t *testing.T) {
   305  	t.Parallel()
   306  
   307  	mn := _NewManagedNetwork()
   308  	mockNode := &mockManagedNode{}
   309  	mockNodeWithError := &mockManagedNodeWithError{
   310  		mockManagedNode: *mockNode,
   311  	}
   312  
   313  	// Inject the node with an error into the healthyNodes slice
   314  	mn.healthyNodes = append(mn.healthyNodes, mockNodeWithError)
   315  
   316  	err := mn._Close()
   317  	require.Error(t, err)
   318  	require.Equal(t, "closing error", err.Error())
   319  }
   320  
   321  func TestUnitManagedNetworkSetTransportSecurityWithError(t *testing.T) {
   322  	t.Parallel()
   323  
   324  	mn := _NewManagedNetwork()
   325  	mockNode := &mockManagedNode{}
   326  	mockNodeWithError := &mockManagedNodeWithError{
   327  		mockManagedNode: *mockNode,
   328  	}
   329  
   330  	// Inject the node with an error into the healthyNodes slice
   331  	mn.healthyNodes = append(mn.healthyNodes, mockNodeWithError)
   332  
   333  	// Attempt to set the transport security
   334  	err := mn._SetTransportSecurity(true)
   335  	require.Error(t, err)
   336  	require.Equal(t, "closing error", err.Error())
   337  }
   338  
   339  func TestUnitSetNetwork_NodeRemoved(t *testing.T) {
   340  	t.Parallel()
   341  
   342  	mn := _NewManagedNetwork()
   343  	mockNodes := newMockNodes()
   344  	err := mn._SetNetwork(mockNodes)
   345  	require.NoError(t, err)
   346  
   347  	// Remove a node from the mockNodes map
   348  	removedNodeKey := "node1:50211"
   349  	removedNode := mockNodes[removedNodeKey]
   350  	delete(mockNodes, removedNodeKey)
   351  
   352  	err = mn._SetNetwork(mockNodes)
   353  	require.NoError(t, err)
   354  
   355  	// Check if the node was removed from the _ManagedNetwork
   356  	require.Equal(t, 2, len(mn.nodes))
   357  	for _, node := range mn.nodes {
   358  		require.NotEqual(t, removedNode._GetAddress(), node._GetAddress())
   359  	}
   360  }
   361  
   362  func TestUnitSetNetwork_NodeAdded(t *testing.T) {
   363  	t.Parallel()
   364  
   365  	mn := _NewManagedNetwork()
   366  	mockNodes := newMockNodes()
   367  	err := mn._SetNetwork(mockNodes)
   368  	require.NoError(t, err)
   369  
   370  	// Add a new node to the mockNodes map
   371  	newNodeKey := "node2:50212"
   372  	address4, _ := _ManagedNodeAddressFromString("node2:50212")
   373  	newNode := &mockManagedNode{address: address4, healthy: true}
   374  	mockNodes[newNodeKey] = newNode
   375  
   376  	err = mn._SetNetwork(mockNodes)
   377  	require.NoError(t, err)
   378  
   379  	// Check if the new node was added to the _ManagedNetwork
   380  	require.Equal(t, 4, len(mn.nodes))
   381  	foundNewNode := false
   382  	for _, node := range mn.nodes {
   383  		if node._GetAddress() == newNode._GetAddress() {
   384  			foundNewNode = true
   385  		}
   386  	}
   387  	require.True(t, foundNewNode)
   388  }
   389  
   390  func TestUnitSetNetworkRemoveAllNodes(t *testing.T) {
   391  	t.Parallel()
   392  
   393  	mn := _NewManagedNetwork()
   394  	mockNodes := newMockNodes()
   395  	err := mn._SetNetwork(mockNodes)
   396  	require.NoError(t, err)
   397  
   398  	// Remove all nodes from the mockNodes map
   399  	for key := range mockNodes {
   400  		delete(mockNodes, key)
   401  	}
   402  
   403  	// Set up the new network without any nodes
   404  	err = mn._SetNetwork(mockNodes)
   405  	require.NoError(t, err)
   406  
   407  	// Check if there are no nodes in the _ManagedNetwork
   408  	require.Equal(t, 0, len(mn.nodes))
   409  }
   410  
   411  func TestUnitReadmitNodes_NodeReadmitted(t *testing.T) {
   412  	t.Parallel()
   413  
   414  	mn := _NewManagedNetwork()
   415  	mockNodes := newMockNodes()
   416  	err := mn._SetNetwork(mockNodes)
   417  	require.NoError(t, err)
   418  
   419  	unhealthyNodeKey := "node1:50211"
   420  	unhealthyNode := mockNodes[unhealthyNodeKey].(*mockManagedNode)
   421  	mn.removeNodeFromHealthyNodes(unhealthyNode)
   422  	unhealthyNode.healthy = true
   423  
   424  	// Set readmit time for the unhealthy node to a time before now
   425  	pastTime := time.Now().Add(-1 * time.Minute)
   426  	unhealthyNode.readmitTime = &pastTime
   427  
   428  	// Call _ReadmitNodes to readmit healthy nodes
   429  	mn._ReadmitNodes()
   430  
   431  	// Check if the previously unhealthy node is now in the healthyNodes list
   432  	found := false
   433  	for _, node := range mn.healthyNodes {
   434  		if node._GetAddress() == unhealthyNodeKey {
   435  			found = true
   436  			break
   437  		}
   438  	}
   439  	require.True(t, found, "node1:50211 should be present in the healthyNodes list after readmission")
   440  }
   441  
   442  func TestUnitReadmitNodes_NodeNotReadmitted(t *testing.T) {
   443  	t.Parallel()
   444  
   445  	mn := _NewManagedNetwork()
   446  	mockNodes := newMockNodes()
   447  	err := mn._SetNetwork(mockNodes)
   448  	require.NoError(t, err)
   449  
   450  	unhealthyNodeKey := "node1:50212"
   451  	unhealthyNode := mockNodes[unhealthyNodeKey].(*mockManagedNode)
   452  	unhealthyNode.healthy = false
   453  	mn.removeNodeFromHealthyNodes(unhealthyNode)
   454  
   455  	// Set readmit time for the unhealthy node to a time in the future
   456  	futureTime := time.Now().Add(1 * time.Hour)
   457  	unhealthyNode.readmitTime = &futureTime
   458  
   459  	// Call _ReadmitNodes
   460  	mn._ReadmitNodes()
   461  
   462  	// Check if the unhealthy node is not present in the healthyNodes list
   463  	found := false
   464  	for _, node := range mn.healthyNodes {
   465  		if node._GetAddress() == unhealthyNodeKey {
   466  			found = true
   467  			break
   468  		}
   469  	}
   470  	require.False(t, found, "node1:50212 should not be present in the healthyNodes list since its readmit time is in the future")
   471  }
   472  
   473  func TestUnitReadmitNodes_UpdateEarliestReadmitTime(t *testing.T) {
   474  	t.Parallel()
   475  
   476  	mn := _NewManagedNetwork()
   477  	mockNodes := newMockNodes()
   478  	err := mn._SetNetwork(mockNodes)
   479  	require.NoError(t, err)
   480  
   481  	// Make a node unhealthy and set its readmit time to a future time, before the minNodeReadmitPeriod
   482  	unhealthyNodeKey := "node1:50211"
   483  	unhealthyNode := mockNodes[unhealthyNodeKey].(*mockManagedNode)
   484  	unhealthyNode.healthy = false
   485  	mn.removeNodeFromHealthyNodes(unhealthyNode)
   486  
   487  	futureReadmitTime := time.Now().Add(3 * time.Second) // Assuming minNodeReadmitPeriod is greater than 3 seconds
   488  	unhealthyNode.readmitTime = &futureReadmitTime
   489  
   490  	// Call _ReadmitNodes
   491  	mn._ReadmitNodes()
   492  
   493  	// Check if the unhealthy node is not present in the healthyNodes list
   494  	found := false
   495  	for _, node := range mn.healthyNodes {
   496  		if node._GetAddress() == unhealthyNodeKey {
   497  			found = true
   498  			break
   499  		}
   500  	}
   501  	require.False(t, found, "node1:50211 should not be present in the healthyNodes list since its readmit time is in the future")
   502  
   503  	// Check if the earliestReadmitTime is updated to now.Add(this.minNodeReadmitPeriod)
   504  	require.WithinDuration(t, futureReadmitTime.Add(mn.minNodeReadmitPeriod), mn.earliestReadmitTime, 5*time.Second)
   505  }
   506  
   507  func TestUnitGetNumberOfNodesForTransaction_Default(t *testing.T) {
   508  	t.Parallel()
   509  
   510  	mn := _NewManagedNetwork()
   511  	mockNodes := newMockNodes()
   512  	err := mn._SetNetwork(mockNodes)
   513  	require.NoError(t, err)
   514  
   515  	numNodes := mn._GetNumberOfNodesForTransaction()
   516  
   517  	// Default behavior: (len(this.network) + 3 - 1) / 3
   518  	expectedNumNodes := (len(mockNodes) + 3 - 1) / 3
   519  	require.Equal(t, expectedNumNodes, numNodes)
   520  }
   521  
   522  func TestUnitGetNumberOfNodesForTransaction_MaxNodesPerTransaction(t *testing.T) {
   523  	t.Parallel()
   524  
   525  	mn := _NewManagedNetwork()
   526  	mockNodes := newMockNodes()
   527  	err := mn._SetNetwork(mockNodes)
   528  	require.NoError(t, err)
   529  
   530  	maxNodes := 2
   531  	mn._SetMaxNodesPerTransaction(maxNodes)
   532  
   533  	numNodes := mn._GetNumberOfNodesForTransaction()
   534  
   535  	// If maxNodesPerTransaction is set, the number of nodes should be the minimum of maxNodesPerTransaction and the number of nodes in the network
   536  	expectedNumNodes := int(math.Min(float64(maxNodes), float64(len(mockNodes))))
   537  	require.Equal(t, expectedNumNodes, numNodes)
   538  }
   539  
   540  func TestUnitGetNumberOfNodesForTransaction_MaxNodesGreaterThanNetworkSize(t *testing.T) {
   541  	t.Parallel()
   542  
   543  	mn := _NewManagedNetwork()
   544  	mockNodes := newMockNodes()
   545  	err := mn._SetNetwork(mockNodes)
   546  	require.NoError(t, err)
   547  
   548  	maxNodes := 9
   549  	mn._SetMaxNodesPerTransaction(maxNodes)
   550  
   551  	numNodes := mn._GetNumberOfNodesForTransaction()
   552  
   553  	expectedNumNodes := int(math.Min(float64(maxNodes), float64(len(mockNodes))))
   554  	require.Equal(t, expectedNumNodes, numNodes)
   555  }
   556  
   557  func TestUnitGetNumberOfNodesForTransaction_MaxNodesNotSet(t *testing.T) {
   558  	t.Parallel()
   559  
   560  	mn := _NewManagedNetwork()
   561  	mockNodes := newMockNodes()
   562  	err := mn._SetNetwork(mockNodes)
   563  	require.NoError(t, err)
   564  
   565  	numNodes := mn._GetNumberOfNodesForTransaction()
   566  	// 1/3 of the network size
   567  	require.Equal(t, 1, numNodes)
   568  }
   569  
   570  func TestUnitGetNode(t *testing.T) {
   571  	t.Parallel()
   572  
   573  	mn := _NewManagedNetwork()
   574  	mockNodes := newMockNodes()
   575  	err := mn._SetNetwork(mockNodes)
   576  	require.NoError(t, err)
   577  
   578  	// Ensure that there are healthy nodes in the network
   579  	require.NotEqual(t, 0, len(mn.healthyNodes))
   580  
   581  	// Get a random node from the managed network
   582  	node := mn._GetNode()
   583  
   584  	// Check if the returned node is not nil
   585  	require.NotNil(t, node)
   586  
   587  	// Check if the returned node is one of the healthy nodes in the managed network
   588  	found := false
   589  	for _, healthyNode := range mn.healthyNodes {
   590  		if node._GetAddress() == healthyNode._GetAddress() {
   591  			found = true
   592  			break
   593  		}
   594  	}
   595  	require.True(t, found, "The returned node should be one of the healthy nodes in the managed network")
   596  }
   597  
   598  func TestUnitGetNodePanicNoHealthyNodes(t *testing.T) {
   599  	t.Parallel()
   600  
   601  	mn := _NewManagedNetwork()
   602  	mockNodes := newMockNodes()
   603  	err := mn._SetNetwork(mockNodes)
   604  	require.NoError(t, err)
   605  
   606  	// Mark all nodes as unhealthy and set their readmit time in the future
   607  	for _, node := range mockNodes {
   608  		node.(*mockManagedNode).healthy = false
   609  		readmitTime := time.Now().Add(1 * time.Minute)
   610  		node.(*mockManagedNode).readmitTime = &readmitTime
   611  	}
   612  
   613  	// Update the network with unhealthy nodes
   614  	err = mn._SetNetwork(mockNodes)
   615  	require.NoError(t, err)
   616  
   617  	// Ensure that there are no healthy nodes in the network
   618  	require.Equal(t, 0, len(mn.healthyNodes))
   619  
   620  	// Check if calling _GetNode() panics when there are no healthy nodes
   621  	defer func() {
   622  		if r := recover(); r != nil {
   623  			panicValue, ok := r.(string)
   624  			require.True(t, ok, "Panic value should be a string")
   625  			require.Equal(t, "failed to find a healthy working node", panicValue)
   626  		}
   627  	}()
   628  
   629  	mn._GetNode()
   630  	require.Fail(t, "Expected _GetNode to panic")
   631  }