sigs.k8s.io/cluster-api@v1.7.1/internal/controllers/machineset/machineset_delete_policy_test.go (about)

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package machineset
    18  
    19  import (
    20  	"fmt"
    21  	"testing"
    22  
    23  	. "github.com/onsi/gomega"
    24  	corev1 "k8s.io/api/core/v1"
    25  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    26  	"k8s.io/apimachinery/pkg/util/rand"
    27  
    28  	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
    29  	capierrors "sigs.k8s.io/cluster-api/errors"
    30  )
    31  
    32  func TestMachineToDelete(t *testing.T) {
    33  	msg := "something wrong with the machine"
    34  	now := metav1.Now()
    35  	nodeRef := &corev1.ObjectReference{Name: "some-node"}
    36  	healthyMachine := &clusterv1.Machine{Status: clusterv1.MachineStatus{NodeRef: nodeRef}}
    37  	mustDeleteMachine := &clusterv1.Machine{
    38  		ObjectMeta: metav1.ObjectMeta{DeletionTimestamp: &now},
    39  		Status:     clusterv1.MachineStatus{NodeRef: nodeRef},
    40  	}
    41  	betterDeleteMachine := &clusterv1.Machine{
    42  		Status: clusterv1.MachineStatus{FailureMessage: &msg, NodeRef: nodeRef},
    43  	}
    44  	deleteMachineWithMachineAnnotation := &clusterv1.Machine{
    45  		ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{clusterv1.DeleteMachineAnnotation: ""}},
    46  		Status:     clusterv1.MachineStatus{NodeRef: nodeRef},
    47  	}
    48  	deleteMachineWithoutNodeRef := &clusterv1.Machine{}
    49  	nodeHealthyConditionFalseMachine := &clusterv1.Machine{
    50  		Status: clusterv1.MachineStatus{
    51  			NodeRef: nodeRef,
    52  			Conditions: clusterv1.Conditions{
    53  				{
    54  					Type:   clusterv1.MachineNodeHealthyCondition,
    55  					Status: corev1.ConditionFalse,
    56  				},
    57  			},
    58  		},
    59  	}
    60  	nodeHealthyConditionUnknownMachine := &clusterv1.Machine{
    61  		Status: clusterv1.MachineStatus{
    62  			NodeRef: nodeRef,
    63  			Conditions: clusterv1.Conditions{
    64  				{
    65  					Type:   clusterv1.MachineNodeHealthyCondition,
    66  					Status: corev1.ConditionUnknown,
    67  				},
    68  			},
    69  		},
    70  	}
    71  
    72  	tests := []struct {
    73  		desc     string
    74  		machines []*clusterv1.Machine
    75  		diff     int
    76  		expect   []*clusterv1.Machine
    77  	}{
    78  		{
    79  			desc: "func=randomDeletePolicy, diff=0",
    80  			diff: 0,
    81  			machines: []*clusterv1.Machine{
    82  				healthyMachine,
    83  			},
    84  			expect: []*clusterv1.Machine{},
    85  		},
    86  		{
    87  			desc: "func=randomDeletePolicy, diff>len(machines)",
    88  			diff: 2,
    89  			machines: []*clusterv1.Machine{
    90  				healthyMachine,
    91  			},
    92  			expect: []*clusterv1.Machine{
    93  				healthyMachine,
    94  			},
    95  		},
    96  		{
    97  			desc: "func=randomDeletePolicy, diff>betterDelete",
    98  			diff: 2,
    99  			machines: []*clusterv1.Machine{
   100  				healthyMachine,
   101  				betterDeleteMachine,
   102  				healthyMachine,
   103  			},
   104  			expect: []*clusterv1.Machine{
   105  				betterDeleteMachine,
   106  				healthyMachine,
   107  			},
   108  		},
   109  		{
   110  			desc: "func=randomDeletePolicy, diff<betterDelete",
   111  			diff: 2,
   112  			machines: []*clusterv1.Machine{
   113  				healthyMachine,
   114  				betterDeleteMachine,
   115  				betterDeleteMachine,
   116  				betterDeleteMachine,
   117  			},
   118  			expect: []*clusterv1.Machine{
   119  				betterDeleteMachine,
   120  				betterDeleteMachine,
   121  			},
   122  		},
   123  		{
   124  			desc: "func=randomDeletePolicy, diff<=mustDelete",
   125  			diff: 2,
   126  			machines: []*clusterv1.Machine{
   127  				healthyMachine,
   128  				mustDeleteMachine,
   129  				betterDeleteMachine,
   130  				mustDeleteMachine,
   131  			},
   132  			expect: []*clusterv1.Machine{
   133  				mustDeleteMachine,
   134  				mustDeleteMachine,
   135  			},
   136  		},
   137  		{
   138  			desc: "func=randomDeletePolicy, diff<=mustDelete+betterDelete",
   139  			diff: 2,
   140  			machines: []*clusterv1.Machine{
   141  				healthyMachine,
   142  				mustDeleteMachine,
   143  				healthyMachine,
   144  				betterDeleteMachine,
   145  			},
   146  			expect: []*clusterv1.Machine{
   147  				mustDeleteMachine,
   148  				betterDeleteMachine,
   149  			},
   150  		},
   151  		{
   152  			desc: "func=randomDeletePolicy, diff<=mustDelete+betterDelete+couldDelete",
   153  			diff: 2,
   154  			machines: []*clusterv1.Machine{
   155  				healthyMachine,
   156  				mustDeleteMachine,
   157  				healthyMachine,
   158  			},
   159  			expect: []*clusterv1.Machine{
   160  				mustDeleteMachine,
   161  				healthyMachine,
   162  			},
   163  		},
   164  		{
   165  			desc: "func=randomDeletePolicy, diff>betterDelete",
   166  			diff: 2,
   167  			machines: []*clusterv1.Machine{
   168  				healthyMachine,
   169  				betterDeleteMachine,
   170  				healthyMachine,
   171  			},
   172  			expect: []*clusterv1.Machine{
   173  				betterDeleteMachine,
   174  				healthyMachine,
   175  			},
   176  		},
   177  		{
   178  			desc: "func=randomDeletePolicy, DeleteMachineAnnotation, diff=1",
   179  			diff: 1,
   180  			machines: []*clusterv1.Machine{
   181  				healthyMachine,
   182  				deleteMachineWithMachineAnnotation,
   183  				healthyMachine,
   184  			},
   185  			expect: []*clusterv1.Machine{
   186  				deleteMachineWithMachineAnnotation,
   187  			},
   188  		},
   189  		{
   190  			desc: "func=randomDeletePolicy, MachineWithNoNodeRef, diff=1",
   191  			diff: 1,
   192  			machines: []*clusterv1.Machine{
   193  				healthyMachine,
   194  				deleteMachineWithoutNodeRef,
   195  				healthyMachine,
   196  			},
   197  			expect: []*clusterv1.Machine{
   198  				deleteMachineWithoutNodeRef,
   199  			},
   200  		},
   201  		{
   202  			desc: "func=randomDeletePolicy, NodeHealthyConditionFalseMachine, diff=1",
   203  			diff: 1,
   204  			machines: []*clusterv1.Machine{
   205  				healthyMachine,
   206  				nodeHealthyConditionFalseMachine,
   207  				healthyMachine,
   208  			},
   209  			expect: []*clusterv1.Machine{
   210  				nodeHealthyConditionFalseMachine,
   211  			},
   212  		},
   213  		{
   214  			desc: "func=randomDeletePolicy, NodeHealthyConditionUnknownMachine, diff=1",
   215  			diff: 1,
   216  			machines: []*clusterv1.Machine{
   217  				healthyMachine,
   218  				nodeHealthyConditionUnknownMachine,
   219  				healthyMachine,
   220  			},
   221  			expect: []*clusterv1.Machine{
   222  				nodeHealthyConditionUnknownMachine,
   223  			},
   224  		},
   225  	}
   226  
   227  	for _, test := range tests {
   228  		t.Run(test.desc, func(t *testing.T) {
   229  			g := NewWithT(t)
   230  
   231  			result := getMachinesToDeletePrioritized(test.machines, test.diff, randomDeletePolicy)
   232  			g.Expect(result).To(BeComparableTo(test.expect))
   233  		})
   234  	}
   235  }
   236  
   237  func TestMachineNewestDelete(t *testing.T) {
   238  	currentTime := metav1.Now()
   239  	statusError := capierrors.MachineStatusError("I'm unhealthy!")
   240  	nodeRef := &corev1.ObjectReference{Name: "some-node"}
   241  	mustDeleteMachine := &clusterv1.Machine{
   242  		ObjectMeta: metav1.ObjectMeta{DeletionTimestamp: &currentTime},
   243  		Status:     clusterv1.MachineStatus{NodeRef: nodeRef},
   244  	}
   245  	newest := &clusterv1.Machine{
   246  		ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -1))},
   247  		Status:     clusterv1.MachineStatus{NodeRef: nodeRef},
   248  	}
   249  	secondNewest := &clusterv1.Machine{
   250  		ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -5))},
   251  		Status:     clusterv1.MachineStatus{NodeRef: nodeRef},
   252  	}
   253  	secondOldest := &clusterv1.Machine{
   254  		ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -10))},
   255  		Status:     clusterv1.MachineStatus{NodeRef: nodeRef},
   256  	}
   257  	oldest := &clusterv1.Machine{
   258  		ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -10))},
   259  		Status:     clusterv1.MachineStatus{NodeRef: nodeRef},
   260  	}
   261  	deleteMachineWithMachineAnnotation := &clusterv1.Machine{
   262  		ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{clusterv1.DeleteMachineAnnotation: ""}, CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -10))},
   263  		Status:     clusterv1.MachineStatus{NodeRef: nodeRef},
   264  	}
   265  	unhealthyMachine := &clusterv1.Machine{
   266  		ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -10))},
   267  		Status:     clusterv1.MachineStatus{FailureReason: &statusError, NodeRef: nodeRef},
   268  	}
   269  	deleteMachineWithoutNodeRef := &clusterv1.Machine{
   270  		ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -1))},
   271  	}
   272  	nodeHealthyConditionFalseMachine := &clusterv1.Machine{
   273  		ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -10))},
   274  		Status: clusterv1.MachineStatus{
   275  			NodeRef: nodeRef,
   276  			Conditions: clusterv1.Conditions{
   277  				{
   278  					Type:   clusterv1.MachineNodeHealthyCondition,
   279  					Status: corev1.ConditionFalse,
   280  				},
   281  			},
   282  		},
   283  	}
   284  	nodeHealthyConditionUnknownMachine := &clusterv1.Machine{
   285  		ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -10))},
   286  		Status: clusterv1.MachineStatus{
   287  			NodeRef: nodeRef,
   288  			Conditions: clusterv1.Conditions{
   289  				{
   290  					Type:   clusterv1.MachineNodeHealthyCondition,
   291  					Status: corev1.ConditionUnknown,
   292  				},
   293  			},
   294  		},
   295  	}
   296  
   297  	tests := []struct {
   298  		desc     string
   299  		machines []*clusterv1.Machine
   300  		diff     int
   301  		expect   []*clusterv1.Machine
   302  	}{
   303  		{
   304  			desc: "func=newestDeletePriority, diff=1",
   305  			diff: 1,
   306  			machines: []*clusterv1.Machine{
   307  				secondNewest, oldest, secondOldest, mustDeleteMachine, newest,
   308  			},
   309  			expect: []*clusterv1.Machine{mustDeleteMachine},
   310  		},
   311  		{
   312  			desc: "func=newestDeletePriority, diff=2",
   313  			diff: 2,
   314  			machines: []*clusterv1.Machine{
   315  				secondNewest, oldest, mustDeleteMachine, secondOldest, newest,
   316  			},
   317  			expect: []*clusterv1.Machine{mustDeleteMachine, newest},
   318  		},
   319  		{
   320  			desc: "func=newestDeletePriority, diff=3",
   321  			diff: 3,
   322  			machines: []*clusterv1.Machine{
   323  				secondNewest, mustDeleteMachine, oldest, secondOldest, newest,
   324  			},
   325  			expect: []*clusterv1.Machine{mustDeleteMachine, newest, secondNewest},
   326  		},
   327  		{
   328  			desc: "func=newestDeletePriority, diff=1 (DeleteMachineAnnotation)",
   329  			diff: 1,
   330  			machines: []*clusterv1.Machine{
   331  				secondNewest, oldest, secondOldest, newest, deleteMachineWithMachineAnnotation,
   332  			},
   333  			expect: []*clusterv1.Machine{deleteMachineWithMachineAnnotation},
   334  		},
   335  		{
   336  			desc: "func=newestDeletePriority, diff=1 (deleteMachineWithoutNodeRef)",
   337  			diff: 1,
   338  			machines: []*clusterv1.Machine{
   339  				secondNewest, oldest, secondOldest, newest, deleteMachineWithoutNodeRef,
   340  			},
   341  			expect: []*clusterv1.Machine{deleteMachineWithoutNodeRef},
   342  		},
   343  		{
   344  			desc: "func=newestDeletePriority, diff=1 (unhealthy)",
   345  			diff: 1,
   346  			machines: []*clusterv1.Machine{
   347  				secondNewest, oldest, secondOldest, newest, unhealthyMachine,
   348  			},
   349  			expect: []*clusterv1.Machine{unhealthyMachine},
   350  		},
   351  		{
   352  			desc: "func=newestDeletePriority, diff=1 (nodeHealthyConditionFalseMachine)",
   353  			diff: 1,
   354  			machines: []*clusterv1.Machine{
   355  				secondNewest, oldest, secondOldest, newest, nodeHealthyConditionFalseMachine,
   356  			},
   357  			expect: []*clusterv1.Machine{nodeHealthyConditionFalseMachine},
   358  		},
   359  		{
   360  			desc: "func=newestDeletePriority, diff=1 (nodeHealthyConditionUnknownMachine)",
   361  			diff: 1,
   362  			machines: []*clusterv1.Machine{
   363  				secondNewest, oldest, secondOldest, newest, nodeHealthyConditionUnknownMachine,
   364  			},
   365  			expect: []*clusterv1.Machine{nodeHealthyConditionUnknownMachine},
   366  		},
   367  	}
   368  
   369  	for _, test := range tests {
   370  		t.Run(test.desc, func(t *testing.T) {
   371  			g := NewWithT(t)
   372  
   373  			result := getMachinesToDeletePrioritized(test.machines, test.diff, newestDeletePriority)
   374  			g.Expect(result).To(BeComparableTo(test.expect))
   375  		})
   376  	}
   377  }
   378  
   379  func TestMachineOldestDelete(t *testing.T) {
   380  	currentTime := metav1.Now()
   381  	statusError := capierrors.MachineStatusError("I'm unhealthy!")
   382  	nodeRef := &corev1.ObjectReference{Name: "some-node"}
   383  	empty := &clusterv1.Machine{
   384  		Status: clusterv1.MachineStatus{NodeRef: nodeRef},
   385  	}
   386  	newest := &clusterv1.Machine{
   387  		ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -1))},
   388  		Status:     clusterv1.MachineStatus{NodeRef: nodeRef},
   389  	}
   390  	secondNewest := &clusterv1.Machine{
   391  		ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -5))},
   392  		Status:     clusterv1.MachineStatus{NodeRef: nodeRef},
   393  	}
   394  	secondOldest := &clusterv1.Machine{
   395  		ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -10))},
   396  		Status:     clusterv1.MachineStatus{NodeRef: nodeRef},
   397  	}
   398  	oldest := &clusterv1.Machine{
   399  		ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -10))},
   400  		Status:     clusterv1.MachineStatus{NodeRef: nodeRef},
   401  	}
   402  	deleteMachineWithMachineAnnotation := &clusterv1.Machine{
   403  		ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{clusterv1.DeleteMachineAnnotation: ""}, CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -10))},
   404  		Status:     clusterv1.MachineStatus{NodeRef: nodeRef},
   405  	}
   406  	unhealthyMachine := &clusterv1.Machine{
   407  		ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -10))},
   408  		Status:     clusterv1.MachineStatus{FailureReason: &statusError, NodeRef: nodeRef},
   409  	}
   410  	mustDeleteMachine := &clusterv1.Machine{
   411  		ObjectMeta: metav1.ObjectMeta{Name: "b", DeletionTimestamp: &currentTime},
   412  		Status:     clusterv1.MachineStatus{NodeRef: nodeRef},
   413  	}
   414  	unhealthyMachineA := &clusterv1.Machine{
   415  		ObjectMeta: metav1.ObjectMeta{Name: "a", CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -10))},
   416  		Status:     clusterv1.MachineStatus{FailureReason: &statusError, NodeRef: nodeRef},
   417  	}
   418  	unhealthyMachineZ := &clusterv1.Machine{
   419  		ObjectMeta: metav1.ObjectMeta{Name: "z", CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -10))},
   420  		Status:     clusterv1.MachineStatus{FailureReason: &statusError, NodeRef: nodeRef},
   421  	}
   422  	deleteMachineWithoutNodeRef := &clusterv1.Machine{
   423  		ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -10))},
   424  	}
   425  	nodeHealthyConditionFalseMachine := &clusterv1.Machine{
   426  		ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -10))},
   427  		Status: clusterv1.MachineStatus{
   428  			NodeRef: nodeRef,
   429  			Conditions: clusterv1.Conditions{
   430  				{
   431  					Type:   clusterv1.MachineNodeHealthyCondition,
   432  					Status: corev1.ConditionFalse,
   433  				},
   434  			},
   435  		},
   436  	}
   437  	nodeHealthyConditionUnknownMachine := &clusterv1.Machine{
   438  		ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.NewTime(currentTime.Time.AddDate(0, 0, -10))},
   439  		Status: clusterv1.MachineStatus{
   440  			NodeRef: nodeRef,
   441  			Conditions: clusterv1.Conditions{
   442  				{
   443  					Type:   clusterv1.MachineNodeHealthyCondition,
   444  					Status: corev1.ConditionUnknown,
   445  				},
   446  			},
   447  		},
   448  	}
   449  
   450  	tests := []struct {
   451  		desc     string
   452  		machines []*clusterv1.Machine
   453  		diff     int
   454  		expect   []*clusterv1.Machine
   455  	}{
   456  		{
   457  			desc: "func=oldestDeletePriority, diff=1",
   458  			diff: 1,
   459  			machines: []*clusterv1.Machine{
   460  				empty, secondNewest, oldest, secondOldest, newest,
   461  			},
   462  			expect: []*clusterv1.Machine{oldest},
   463  		},
   464  		{
   465  			desc: "func=oldestDeletePriority, diff=2",
   466  			diff: 2,
   467  			machines: []*clusterv1.Machine{
   468  				secondNewest, oldest, secondOldest, newest, empty,
   469  			},
   470  			expect: []*clusterv1.Machine{oldest, secondOldest},
   471  		},
   472  		{
   473  			desc: "func=oldestDeletePriority, diff=3",
   474  			diff: 3,
   475  			machines: []*clusterv1.Machine{
   476  				secondNewest, oldest, secondOldest, newest, empty,
   477  			},
   478  			expect: []*clusterv1.Machine{oldest, secondOldest, secondNewest},
   479  		},
   480  		{
   481  			desc: "func=oldestDeletePriority, diff=4",
   482  			diff: 4,
   483  			machines: []*clusterv1.Machine{
   484  				secondNewest, oldest, secondOldest, newest, empty,
   485  			},
   486  			expect: []*clusterv1.Machine{oldest, secondOldest, secondNewest, newest},
   487  		},
   488  		{
   489  			desc: "func=oldestDeletePriority, diff=1 (DeleteMachineAnnotation)",
   490  			diff: 1,
   491  			machines: []*clusterv1.Machine{
   492  				empty, secondNewest, oldest, secondOldest, newest, deleteMachineWithMachineAnnotation,
   493  			},
   494  			expect: []*clusterv1.Machine{deleteMachineWithMachineAnnotation},
   495  		},
   496  		{
   497  			desc: "func=oldestDeletePriority, diff=1 (deleteMachineWithoutNodeRef)",
   498  			diff: 1,
   499  			machines: []*clusterv1.Machine{
   500  				empty, secondNewest, oldest, secondOldest, newest, deleteMachineWithoutNodeRef,
   501  			},
   502  			expect: []*clusterv1.Machine{deleteMachineWithoutNodeRef},
   503  		},
   504  		{
   505  			desc: "func=oldestDeletePriority, diff=1 (unhealthy)",
   506  			diff: 1,
   507  			machines: []*clusterv1.Machine{
   508  				empty, secondNewest, oldest, secondOldest, newest, unhealthyMachine,
   509  			},
   510  			expect: []*clusterv1.Machine{unhealthyMachine},
   511  		},
   512  		{
   513  			desc: "func=oldestDeletePriority, diff=1 (nodeHealthyConditionFalseMachine)",
   514  			diff: 1,
   515  			machines: []*clusterv1.Machine{
   516  				empty, secondNewest, oldest, secondOldest, newest, nodeHealthyConditionFalseMachine,
   517  			},
   518  			expect: []*clusterv1.Machine{nodeHealthyConditionFalseMachine},
   519  		},
   520  		{
   521  			desc: "func=oldestDeletePriority, diff=1 (nodeHealthyConditionUnknownMachine)",
   522  			diff: 1,
   523  			machines: []*clusterv1.Machine{
   524  				empty, secondNewest, oldest, secondOldest, newest, nodeHealthyConditionUnknownMachine,
   525  			},
   526  			expect: []*clusterv1.Machine{nodeHealthyConditionUnknownMachine},
   527  		},
   528  		// these two cases ensures the mustDeleteMachine is always picked regardless of the machine names.
   529  		{
   530  			desc: "func=oldestDeletePriority, diff=1 (unhealthyMachineA)",
   531  			diff: 1,
   532  			machines: []*clusterv1.Machine{
   533  				empty, secondNewest, oldest, secondOldest, newest, mustDeleteMachine, unhealthyMachineA,
   534  			},
   535  			expect: []*clusterv1.Machine{mustDeleteMachine},
   536  		},
   537  		{
   538  			desc: "func=oldestDeletePriority, diff=1 (unhealthyMachineZ)",
   539  			diff: 1,
   540  			machines: []*clusterv1.Machine{
   541  				empty, secondNewest, oldest, secondOldest, newest, mustDeleteMachine, unhealthyMachineZ,
   542  			},
   543  			expect: []*clusterv1.Machine{mustDeleteMachine},
   544  		},
   545  	}
   546  
   547  	for _, test := range tests {
   548  		t.Run(test.desc, func(t *testing.T) {
   549  			g := NewWithT(t)
   550  
   551  			result := getMachinesToDeletePrioritized(test.machines, test.diff, oldestDeletePriority)
   552  			g.Expect(result).To(BeComparableTo(test.expect))
   553  		})
   554  	}
   555  }
   556  
   557  func TestMachineDeleteMultipleSamePriority(t *testing.T) {
   558  	machines := make([]*clusterv1.Machine, 0, 10)
   559  	// All of these machines will have the same delete priority because they all have the "must delete" annotation.
   560  	for i := 0; i < 10; i++ {
   561  		machines = append(machines, &clusterv1.Machine{
   562  			ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("machine-%d", i), Annotations: map[string]string{clusterv1.DeleteMachineAnnotation: "true"}},
   563  		})
   564  	}
   565  
   566  	tests := []struct {
   567  		desc           string
   568  		diff           int
   569  		deletePriority deletePriorityFunc
   570  	}{
   571  		{
   572  			desc:           "multiple with same priority, func=oldestDeletePriority, diff=1",
   573  			diff:           1,
   574  			deletePriority: oldestDeletePriority,
   575  		},
   576  		{
   577  			desc:           "multiple with same priority, func=oldestDeletePriority, diff=5",
   578  			diff:           5,
   579  			deletePriority: oldestDeletePriority,
   580  		},
   581  		{
   582  			desc:           "multiple with same priority, func=oldestDeletePriority, diff=rand",
   583  			diff:           rand.Intn(len(machines)),
   584  			deletePriority: oldestDeletePriority,
   585  		},
   586  		{
   587  			desc:           "multiple with same priority, func=randomDeletePolicy, diff=1",
   588  			diff:           1,
   589  			deletePriority: randomDeletePolicy,
   590  		},
   591  		{
   592  			desc:           "multiple with same priority, func=randomDeletePolicy, diff=5",
   593  			diff:           5,
   594  			deletePriority: randomDeletePolicy,
   595  		},
   596  		{
   597  			desc:           "multiple with same priority, func=randomDeletePolicy, diff=rand",
   598  			diff:           rand.Intn(len(machines)),
   599  			deletePriority: randomDeletePolicy,
   600  		},
   601  		{
   602  			desc:           "multiple with same priority, func=newestDeletePriority, diff=1",
   603  			diff:           1,
   604  			deletePriority: newestDeletePriority,
   605  		},
   606  		{
   607  			desc:           "multiple with same priority, func=newestDeletePriority, diff=5",
   608  			diff:           5,
   609  			deletePriority: newestDeletePriority,
   610  		},
   611  		{
   612  			desc:           "multiple with same priority, func=newestDeletePriority, diff=rand",
   613  			diff:           rand.Intn(len(machines)),
   614  			deletePriority: newestDeletePriority,
   615  		},
   616  		{
   617  			desc:           "multiple with same priority, func=randomDeletePolicy, diff=1",
   618  			diff:           1,
   619  			deletePriority: randomDeletePolicy,
   620  		},
   621  		{
   622  			desc:           "multiple with same priority, func=randomDeletePolicy, diff=5",
   623  			diff:           5,
   624  			deletePriority: randomDeletePolicy,
   625  		},
   626  		{
   627  			desc:           "multiple with same priority, func=randomDeletePolicy, diff=rand",
   628  			diff:           rand.Intn(len(machines)),
   629  			deletePriority: randomDeletePolicy,
   630  		},
   631  	}
   632  
   633  	for _, test := range tests {
   634  		t.Run(test.desc, func(t *testing.T) {
   635  			g := NewWithT(t)
   636  
   637  			order := rand.Perm(len(machines))
   638  			shuffledMachines := make([]*clusterv1.Machine, len(machines))
   639  			for i, j := range order {
   640  				shuffledMachines[i] = machines[j]
   641  			}
   642  
   643  			result := getMachinesToDeletePrioritized(shuffledMachines, test.diff, test.deletePriority)
   644  			g.Expect(result).To(BeComparableTo(machines[:test.diff]))
   645  		})
   646  	}
   647  }
   648  
   649  func TestIsMachineHealthy(t *testing.T) {
   650  	nodeRef := &corev1.ObjectReference{Name: "some-node"}
   651  	statusError := capierrors.MachineStatusError("I'm unhealthy!")
   652  	msg := "something wrong with the machine"
   653  
   654  	tests := []struct {
   655  		desc    string
   656  		machine *clusterv1.Machine
   657  		expect  bool
   658  	}{
   659  		{
   660  			desc:    "when it has no NodeRef",
   661  			machine: &clusterv1.Machine{},
   662  			expect:  false,
   663  		},
   664  		{
   665  			desc: "when it has a FailureReason",
   666  			machine: &clusterv1.Machine{
   667  				Status: clusterv1.MachineStatus{FailureReason: &statusError, NodeRef: nodeRef},
   668  			},
   669  			expect: false,
   670  		},
   671  		{
   672  			desc: "when it has a FailureMessage",
   673  			machine: &clusterv1.Machine{
   674  				Status: clusterv1.MachineStatus{FailureMessage: &msg, NodeRef: nodeRef},
   675  			},
   676  			expect: false,
   677  		},
   678  		{
   679  			desc: "when nodeHealthyCondition is false",
   680  			machine: &clusterv1.Machine{
   681  				Status: clusterv1.MachineStatus{
   682  					NodeRef: nodeRef,
   683  					Conditions: clusterv1.Conditions{
   684  						{
   685  							Type:   clusterv1.MachineNodeHealthyCondition,
   686  							Status: corev1.ConditionFalse,
   687  						},
   688  					},
   689  				},
   690  			},
   691  			expect: false,
   692  		},
   693  		{
   694  			desc: "when nodeHealthyCondition is unknown",
   695  			machine: &clusterv1.Machine{
   696  				Status: clusterv1.MachineStatus{
   697  					NodeRef: nodeRef,
   698  					Conditions: clusterv1.Conditions{
   699  						{
   700  							Type:   clusterv1.MachineNodeHealthyCondition,
   701  							Status: corev1.ConditionUnknown,
   702  						},
   703  					},
   704  				},
   705  			},
   706  			expect: false,
   707  		},
   708  		{
   709  			desc: "when all requirements are met for node to be healthy",
   710  			machine: &clusterv1.Machine{
   711  				Status: clusterv1.MachineStatus{NodeRef: nodeRef},
   712  			},
   713  			expect: true,
   714  		},
   715  	}
   716  
   717  	for _, test := range tests {
   718  		t.Run(test.desc, func(t *testing.T) {
   719  			g := NewWithT(t)
   720  
   721  			result := isMachineHealthy(test.machine)
   722  			g.Expect(result).To(Equal(test.expect))
   723  		})
   724  	}
   725  }