github.com/spotahome/redis-operator@v1.2.4/operator/redisfailover/checker_test.go (about)

     1  package redisfailover_test
     2  
     3  import (
     4  	"errors"
     5  	"fmt"
     6  	"testing"
     7  	"time"
     8  
     9  	"github.com/stretchr/testify/assert"
    10  
    11  	appsv1 "k8s.io/api/apps/v1"
    12  	corev1 "k8s.io/api/core/v1"
    13  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    14  
    15  	"github.com/spotahome/redis-operator/log"
    16  	"github.com/spotahome/redis-operator/metrics"
    17  	mRFService "github.com/spotahome/redis-operator/mocks/operator/redisfailover/service"
    18  	mK8SService "github.com/spotahome/redis-operator/mocks/service/k8s"
    19  	rfOperator "github.com/spotahome/redis-operator/operator/redisfailover"
    20  )
    21  
    22  func TestCheckAndHeal(t *testing.T) {
    23  	tests := []struct {
    24  		name                           string
    25  		nMasters                       int
    26  		nRedis                         int
    27  		forceNewMasterNoQrm            bool
    28  		forceNewMasterFirstBoot        bool
    29  		singleMasterTest               bool
    30  		slavesOK                       bool
    31  		sentinelMonitorOK              bool
    32  		sentinelNumberInMemoryOK       bool
    33  		sentinelSlavesNumberInMemoryOK bool
    34  		redisCheckNumberOK             bool
    35  		redisSetMasterOnAllOK          bool
    36  		bootstrapping                  bool
    37  		allowSentinels                 bool
    38  	}{
    39  		{
    40  			name:                           "Everything ok, no need to heal",
    41  			nMasters:                       1,
    42  			nRedis:                         3,
    43  			singleMasterTest:               false,
    44  			forceNewMasterNoQrm:            false,
    45  			forceNewMasterFirstBoot:        false,
    46  			slavesOK:                       true,
    47  			sentinelMonitorOK:              true,
    48  			sentinelNumberInMemoryOK:       true,
    49  			sentinelSlavesNumberInMemoryOK: true,
    50  			redisCheckNumberOK:             true,
    51  			redisSetMasterOnAllOK:          true,
    52  			bootstrapping:                  false,
    53  			allowSentinels:                 false,
    54  		},
    55  		{
    56  			name:                           "Multiple masters",
    57  			nMasters:                       2,
    58  			nRedis:                         3,
    59  			singleMasterTest:               false,
    60  			forceNewMasterNoQrm:            false,
    61  			forceNewMasterFirstBoot:        false,
    62  			slavesOK:                       true,
    63  			sentinelMonitorOK:              true,
    64  			sentinelNumberInMemoryOK:       true,
    65  			sentinelSlavesNumberInMemoryOK: true,
    66  			redisCheckNumberOK:             true,
    67  			redisSetMasterOnAllOK:          true,
    68  			bootstrapping:                  false,
    69  			allowSentinels:                 false,
    70  		},
    71  		{
    72  			name:                           "No masters but wait",
    73  			nMasters:                       0,
    74  			nRedis:                         3,
    75  			singleMasterTest:               false,
    76  			forceNewMasterNoQrm:            false,
    77  			forceNewMasterFirstBoot:        false,
    78  			slavesOK:                       true,
    79  			sentinelMonitorOK:              true,
    80  			sentinelNumberInMemoryOK:       true,
    81  			sentinelSlavesNumberInMemoryOK: true,
    82  			redisCheckNumberOK:             true,
    83  			redisSetMasterOnAllOK:          true,
    84  			bootstrapping:                  false,
    85  			allowSentinels:                 false,
    86  		},
    87  		{
    88  			name:                           "No masters, only one redis available, make master",
    89  			nMasters:                       0,
    90  			nRedis:                         1,
    91  			singleMasterTest:               true,
    92  			forceNewMasterNoQrm:            false,
    93  			forceNewMasterFirstBoot:        false,
    94  			slavesOK:                       true,
    95  			sentinelMonitorOK:              true,
    96  			sentinelNumberInMemoryOK:       true,
    97  			sentinelSlavesNumberInMemoryOK: true,
    98  			redisCheckNumberOK:             true,
    99  			redisSetMasterOnAllOK:          true,
   100  			bootstrapping:                  false,
   101  			allowSentinels:                 false,
   102  		},
   103  		{
   104  			name:                           "No masters,No sentinel quorum set random",
   105  			nMasters:                       0,
   106  			nRedis:                         3,
   107  			singleMasterTest:               false,
   108  			forceNewMasterNoQrm:            true,
   109  			forceNewMasterFirstBoot:        false,
   110  			slavesOK:                       true,
   111  			sentinelMonitorOK:              true,
   112  			sentinelNumberInMemoryOK:       true,
   113  			redisCheckNumberOK:             true,
   114  			redisSetMasterOnAllOK:          true,
   115  			sentinelSlavesNumberInMemoryOK: true,
   116  			allowSentinels:                 false,
   117  		},
   118  		{
   119  			name:                           "No masters,Sentinel Quorum but slave of local host set random",
   120  			nMasters:                       0,
   121  			nRedis:                         3,
   122  			singleMasterTest:               false,
   123  			forceNewMasterNoQrm:            false,
   124  			forceNewMasterFirstBoot:        true,
   125  			slavesOK:                       true,
   126  			sentinelMonitorOK:              true,
   127  			sentinelNumberInMemoryOK:       true,
   128  			redisCheckNumberOK:             true,
   129  			redisSetMasterOnAllOK:          true,
   130  			sentinelSlavesNumberInMemoryOK: true,
   131  			allowSentinels:                 false,
   132  		},
   133  		{
   134  			name:                           "Slaves from master wrong",
   135  			nMasters:                       1,
   136  			nRedis:                         3,
   137  			singleMasterTest:               false,
   138  			forceNewMasterNoQrm:            false,
   139  			forceNewMasterFirstBoot:        false,
   140  			slavesOK:                       false,
   141  			sentinelMonitorOK:              true,
   142  			sentinelNumberInMemoryOK:       true,
   143  			sentinelSlavesNumberInMemoryOK: true,
   144  			redisCheckNumberOK:             true,
   145  			redisSetMasterOnAllOK:          true,
   146  			bootstrapping:                  false,
   147  			allowSentinels:                 false,
   148  		},
   149  		{
   150  			name:                           "Sentinels not pointing correct monitor",
   151  			nMasters:                       1,
   152  			nRedis:                         3,
   153  			singleMasterTest:               false,
   154  			forceNewMasterNoQrm:            false,
   155  			forceNewMasterFirstBoot:        false,
   156  			slavesOK:                       true,
   157  			sentinelMonitorOK:              false,
   158  			sentinelNumberInMemoryOK:       true,
   159  			sentinelSlavesNumberInMemoryOK: true,
   160  			redisCheckNumberOK:             true,
   161  			redisSetMasterOnAllOK:          true,
   162  			bootstrapping:                  false,
   163  			allowSentinels:                 false,
   164  		},
   165  		{
   166  			name:                           "Sentinels with wrong number of sentinels",
   167  			nMasters:                       1,
   168  			nRedis:                         3,
   169  			singleMasterTest:               false,
   170  			forceNewMasterNoQrm:            false,
   171  			forceNewMasterFirstBoot:        false,
   172  			slavesOK:                       true,
   173  			sentinelMonitorOK:              true,
   174  			sentinelNumberInMemoryOK:       false,
   175  			sentinelSlavesNumberInMemoryOK: true,
   176  			redisCheckNumberOK:             true,
   177  			redisSetMasterOnAllOK:          true,
   178  			bootstrapping:                  false,
   179  			allowSentinels:                 false,
   180  		},
   181  		{
   182  			name:                           "Sentinels with wrong number of slaves",
   183  			nMasters:                       1,
   184  			nRedis:                         3,
   185  			singleMasterTest:               false,
   186  			forceNewMasterNoQrm:            false,
   187  			forceNewMasterFirstBoot:        false,
   188  			slavesOK:                       true,
   189  			sentinelMonitorOK:              true,
   190  			sentinelNumberInMemoryOK:       true,
   191  			sentinelSlavesNumberInMemoryOK: false,
   192  			redisCheckNumberOK:             true,
   193  			redisSetMasterOnAllOK:          true,
   194  			bootstrapping:                  false,
   195  			allowSentinels:                 false,
   196  		},
   197  		{
   198  			name:                  "Bootstrapping Mode",
   199  			nMasters:              1,
   200  			nRedis:                3,
   201  			redisCheckNumberOK:    true,
   202  			redisSetMasterOnAllOK: true,
   203  			bootstrapping:         true,
   204  			allowSentinels:        false,
   205  		},
   206  		{
   207  			name:                  "Bootstrapping Mode with failure to check redis number",
   208  			nMasters:              1,
   209  			nRedis:                3,
   210  			redisCheckNumberOK:    false,
   211  			redisSetMasterOnAllOK: true,
   212  			bootstrapping:         true,
   213  			allowSentinels:        false,
   214  		},
   215  		{
   216  			name:                  "Bootstrapping Mode with failure to set master on all",
   217  			nMasters:              1,
   218  			nRedis:                3,
   219  			redisCheckNumberOK:    true,
   220  			redisSetMasterOnAllOK: false,
   221  			bootstrapping:         true,
   222  			allowSentinels:        false,
   223  		},
   224  		{
   225  			name:                           "Bootstrapping Mode that allows sentinels",
   226  			nMasters:                       1,
   227  			nRedis:                         3,
   228  			redisCheckNumberOK:             true,
   229  			redisSetMasterOnAllOK:          true,
   230  			sentinelMonitorOK:              true,
   231  			sentinelNumberInMemoryOK:       true,
   232  			sentinelSlavesNumberInMemoryOK: true,
   233  			bootstrapping:                  true,
   234  			allowSentinels:                 true,
   235  		},
   236  		{
   237  			name:                           "Bootstrapping Mode that allows sentinels sentinel monitor fails",
   238  			nMasters:                       1,
   239  			nRedis:                         3,
   240  			redisCheckNumberOK:             true,
   241  			redisSetMasterOnAllOK:          true,
   242  			sentinelMonitorOK:              false,
   243  			sentinelNumberInMemoryOK:       true,
   244  			sentinelSlavesNumberInMemoryOK: true,
   245  			bootstrapping:                  true,
   246  			allowSentinels:                 true,
   247  		},
   248  		{
   249  			name:                           "Bootstrapping Mode that allows sentinels sentinel with wrong number of sentinels",
   250  			nMasters:                       1,
   251  			nRedis:                         3,
   252  			redisCheckNumberOK:             true,
   253  			redisSetMasterOnAllOK:          true,
   254  			sentinelMonitorOK:              true,
   255  			sentinelNumberInMemoryOK:       false,
   256  			sentinelSlavesNumberInMemoryOK: true,
   257  			bootstrapping:                  true,
   258  			allowSentinels:                 true,
   259  		},
   260  		{
   261  			name:                           "Bootstrapping Mode that allows sentinels sentinel with wrong number of slaves",
   262  			nMasters:                       1,
   263  			nRedis:                         3,
   264  			redisCheckNumberOK:             true,
   265  			redisSetMasterOnAllOK:          true,
   266  			sentinelMonitorOK:              true,
   267  			sentinelNumberInMemoryOK:       true,
   268  			sentinelSlavesNumberInMemoryOK: false,
   269  			bootstrapping:                  true,
   270  			allowSentinels:                 true,
   271  		},
   272  	}
   273  
   274  	for _, test := range tests {
   275  		t.Run(test.name, func(t *testing.T) {
   276  			assert := assert.New(t)
   277  
   278  			allowSentinels := true
   279  			bootstrappingTests := test.bootstrapping
   280  			bootstrapMaster := "127.0.0.1"
   281  			bootstrapMasterPort := "6379"
   282  
   283  			rf := generateRF(false, bootstrappingTests)
   284  			if bootstrappingTests {
   285  				allowSentinels = test.allowSentinels
   286  				rf.Spec.BootstrapNode.AllowSentinels = allowSentinels
   287  			}
   288  			if test.singleMasterTest {
   289  				rf.Spec.Redis.Replicas = 1
   290  			}
   291  
   292  			expErr := false
   293  			continueTests := true
   294  
   295  			master := "0.0.0.0"
   296  			sentinel := "1.1.1.1"
   297  
   298  			config := generateConfig()
   299  			mk := &mK8SService.Services{}
   300  			mrfs := &mRFService.RedisFailoverClient{}
   301  			mrfc := &mRFService.RedisFailoverCheck{}
   302  			mrfh := &mRFService.RedisFailoverHeal{}
   303  
   304  			if test.redisCheckNumberOK {
   305  				mrfc.On("IsRedisRunning", rf).Once().Return(true)
   306  			} else {
   307  				continueTests = false
   308  				mrfc.On("IsRedisRunning", rf).Once().Return(false)
   309  			}
   310  
   311  			if allowSentinels {
   312  				mrfc.On("IsSentinelRunning", rf).Once().Return(true)
   313  			}
   314  
   315  			if bootstrappingTests && continueTests {
   316  				// once to get ips for config update, once for the UpdateRedisesPods go right
   317  				mrfc.On("GetRedisesIPs", rf).Twice().Return([]string{"0.0.0.1", "0.0.0.2", "0.0.0.3"}, nil)
   318  				mrfh.On("SetRedisCustomConfig", "0.0.0.1", rf).Once().Return(nil)
   319  				mrfh.On("SetRedisCustomConfig", "0.0.0.2", rf).Once().Return(nil)
   320  				mrfh.On("SetRedisCustomConfig", "0.0.0.3", rf).Once().Return(nil)
   321  				mrfc.On("CheckRedisSlavesReady", "0.0.0.1", rf).Once().Return(true, nil)
   322  				mrfc.On("CheckRedisSlavesReady", "0.0.0.2", rf).Once().Return(true, nil)
   323  				mrfc.On("CheckRedisSlavesReady", "0.0.0.3", rf).Once().Return(true, nil)
   324  				mrfc.On("GetStatefulSetUpdateRevision", rf).Once().Return("1", nil)
   325  				mrfc.On("GetRedisesSlavesPods", rf).Once().Return([]string{}, nil)
   326  
   327  				if test.redisSetMasterOnAllOK {
   328  					mrfh.On("SetExternalMasterOnAll", bootstrapMaster, bootstrapMasterPort, rf).Once().Return(nil)
   329  				} else {
   330  					expErr = true
   331  					mrfh.On("SetExternalMasterOnAll", bootstrapMaster, bootstrapMasterPort, rf).Once().Return(errors.New(""))
   332  				}
   333  			} else if continueTests {
   334  				mrfc.On("GetNumberMasters", rf).Once().Return(test.nMasters, nil)
   335  				switch test.nMasters {
   336  				case 0:
   337  					//mrfc.On("GetRedisesIPs", rf).Once().Return(make([]string, test.nRedis), nil)
   338  					if rf.Spec.Redis.Replicas == 1 {
   339  						mrfh.On("SetOldestAsMaster", rf).Once().Return(nil)
   340  						continueTests = false
   341  						break
   342  					}
   343  					mrfc.On("GetMaxRedisPodTime", rf).Once().Return(1*time.Hour, nil)
   344  					if test.forceNewMasterNoQrm {
   345  						mrfc.On("CheckSentinelQuorum", rf).Once().Return(1, errors.New(""))
   346  						mrfh.On("SetOldestAsMaster", rf).Once().Return(nil)
   347  					} else if test.forceNewMasterFirstBoot {
   348  						mrfc.On("CheckSentinelQuorum", rf).Once().Return(3, nil)
   349  						mrfc.On("CheckIfMasterLocalhost", rf).Once().Return(true, nil)
   350  						mrfh.On("SetOldestAsMaster", rf).Once().Return(nil)
   351  					} else {
   352  						mrfc.On("CheckSentinelQuorum", rf).Once().Return(3, nil)
   353  						mrfc.On("CheckIfMasterLocalhost", rf).Once().Return(false, nil)
   354  						continueTests = false
   355  					}
   356  
   357  				case 1:
   358  					break
   359  				default:
   360  					// always expect error
   361  					expErr = true
   362  				}
   363  				if !expErr && continueTests {
   364  					mrfc.On("GetMasterIP", rf).Twice().Return(master, nil)
   365  					if test.slavesOK {
   366  						mrfc.On("CheckAllSlavesFromMaster", master, rf).Once().Return(nil)
   367  					} else {
   368  						mrfc.On("CheckAllSlavesFromMaster", master, rf).Once().Return(errors.New(""))
   369  						if test.redisSetMasterOnAllOK {
   370  							mrfh.On("SetMasterOnAll", master, rf).Once().Return(nil)
   371  						} else {
   372  							expErr = true
   373  							mrfh.On("SetMasterOnAll", master, rf).Once().Return(errors.New(""))
   374  						}
   375  
   376  					}
   377  					mrfc.On("GetRedisesIPs", rf).Twice().Return([]string{master}, nil)
   378  					mrfc.On("GetStatefulSetUpdateRevision", rf).Once().Return("1", nil)
   379  					mrfc.On("GetRedisesSlavesPods", rf).Once().Return([]string{}, nil)
   380  					mrfc.On("GetRedisesMasterPod", rf).Once().Return(master, nil)
   381  					mrfc.On("GetRedisRevisionHash", master, rf).Once().Return("1", nil)
   382  					mrfh.On("SetRedisCustomConfig", master, rf).Once().Return(nil)
   383  				}
   384  			}
   385  
   386  			if allowSentinels && !expErr && continueTests {
   387  				mrfc.On("GetSentinelsIPs", rf).Once().Return([]string{sentinel}, nil)
   388  				if test.sentinelMonitorOK {
   389  					if test.bootstrapping {
   390  						mrfc.On("CheckSentinelMonitor", sentinel, bootstrapMaster, bootstrapMasterPort).Once().Return(nil)
   391  					} else {
   392  						mrfc.On("CheckSentinelMonitor", sentinel, master, "0").Once().Return(nil)
   393  					}
   394  				} else {
   395  					if test.bootstrapping {
   396  						mrfc.On("CheckSentinelMonitor", sentinel, bootstrapMaster, bootstrapMasterPort).Once().Return(errors.New(""))
   397  						mrfh.On("NewSentinelMonitorWithPort", sentinel, bootstrapMaster, bootstrapMasterPort, rf).Once().Return(nil)
   398  					} else {
   399  						mrfc.On("CheckSentinelMonitor", sentinel, master, "0").Once().Return(errors.New(""))
   400  						mrfh.On("NewSentinelMonitor", sentinel, master, rf).Once().Return(nil)
   401  					}
   402  				}
   403  				if test.sentinelNumberInMemoryOK {
   404  					mrfc.On("CheckSentinelNumberInMemory", sentinel, rf).Once().Return(nil)
   405  				} else {
   406  					mrfc.On("CheckSentinelNumberInMemory", sentinel, rf).Once().Return(errors.New(""))
   407  					mrfh.On("RestoreSentinel", sentinel).Once().Return(nil)
   408  				}
   409  				if test.sentinelSlavesNumberInMemoryOK {
   410  					mrfc.On("CheckSentinelSlavesNumberInMemory", sentinel, rf).Once().Return(nil)
   411  				} else {
   412  					mrfc.On("CheckSentinelSlavesNumberInMemory", sentinel, rf).Once().Return(errors.New(""))
   413  					mrfh.On("RestoreSentinel", sentinel).Once().Return(nil)
   414  				}
   415  				mrfh.On("SetSentinelCustomConfig", sentinel, rf).Once().Return(nil)
   416  			}
   417  
   418  			handler := rfOperator.NewRedisFailoverHandler(config, mrfs, mrfc, mrfh, mk, metrics.Dummy, log.Dummy)
   419  			err := handler.CheckAndHeal(rf)
   420  
   421  			if expErr {
   422  				assert.Error(err)
   423  			} else {
   424  				assert.NoError(err)
   425  			}
   426  			mrfc.AssertExpectations(t)
   427  			mrfh.AssertExpectations(t)
   428  		})
   429  	}
   430  }
   431  
   432  func TestUpdate(t *testing.T) {
   433  	type podStatus struct {
   434  		pod    corev1.Pod
   435  		ready  bool
   436  		master bool
   437  	}
   438  	tests := []struct {
   439  		name          string
   440  		pods          []podStatus
   441  		ssVersion     string
   442  		errExpected   bool
   443  		bootstrapping bool
   444  		noMaster      bool
   445  	}{
   446  		{
   447  			name: "all ok, no change needed",
   448  			pods: []podStatus{
   449  				{
   450  					pod: corev1.Pod{
   451  						ObjectMeta: metav1.ObjectMeta{
   452  							Name: "slave1",
   453  							Labels: map[string]string{
   454  								appsv1.ControllerRevisionHashLabelKey: "10",
   455  							},
   456  						},
   457  						Status: corev1.PodStatus{
   458  							PodIP: "0.0.0.0",
   459  						},
   460  					},
   461  					master: false,
   462  					ready:  true,
   463  				},
   464  				{
   465  					pod: corev1.Pod{
   466  						ObjectMeta: metav1.ObjectMeta{
   467  							Name: "slave2",
   468  							Labels: map[string]string{
   469  								appsv1.ControllerRevisionHashLabelKey: "10",
   470  							},
   471  						},
   472  						Status: corev1.PodStatus{
   473  							PodIP: "0.0.0.1",
   474  						},
   475  					},
   476  					master: false,
   477  					ready:  true,
   478  				},
   479  				{
   480  					pod: corev1.Pod{
   481  						ObjectMeta: metav1.ObjectMeta{
   482  							Name: "master",
   483  							Labels: map[string]string{
   484  								appsv1.ControllerRevisionHashLabelKey: "10",
   485  							},
   486  						},
   487  						Status: corev1.PodStatus{
   488  							PodIP: "1.1.1.1",
   489  						},
   490  					},
   491  					master: true,
   492  					ready:  true,
   493  				},
   494  			},
   495  			ssVersion:     "10",
   496  			errExpected:   false,
   497  			bootstrapping: false,
   498  		},
   499  		{
   500  			name: "syncing",
   501  			pods: []podStatus{
   502  				{
   503  					pod: corev1.Pod{
   504  						ObjectMeta: metav1.ObjectMeta{
   505  							Name: "slave1",
   506  							Labels: map[string]string{
   507  								appsv1.ControllerRevisionHashLabelKey: "10",
   508  							},
   509  						},
   510  						Status: corev1.PodStatus{
   511  							PodIP: "0.0.0.0",
   512  						},
   513  					},
   514  					master: false,
   515  					ready:  true,
   516  				},
   517  				{
   518  					pod: corev1.Pod{
   519  						ObjectMeta: metav1.ObjectMeta{
   520  							Name: "slave2",
   521  							Labels: map[string]string{
   522  								appsv1.ControllerRevisionHashLabelKey: "10",
   523  							},
   524  						},
   525  						Status: corev1.PodStatus{
   526  							PodIP: "0.0.0.1",
   527  						},
   528  					},
   529  					master: false,
   530  					ready:  false,
   531  				},
   532  				{
   533  					pod: corev1.Pod{
   534  						ObjectMeta: metav1.ObjectMeta{
   535  							Name: "master",
   536  							Labels: map[string]string{
   537  								appsv1.ControllerRevisionHashLabelKey: "10",
   538  							},
   539  						},
   540  						Status: corev1.PodStatus{
   541  							PodIP: "1.1.1.1",
   542  						},
   543  					},
   544  					master: true,
   545  					ready:  true,
   546  				},
   547  			},
   548  			ssVersion:     "10",
   549  			errExpected:   false,
   550  			bootstrapping: false,
   551  		},
   552  		{
   553  			name: "pod version incorrect",
   554  			pods: []podStatus{
   555  				{
   556  					pod: corev1.Pod{
   557  						ObjectMeta: metav1.ObjectMeta{
   558  							Name: "slave1",
   559  							Labels: map[string]string{
   560  								appsv1.ControllerRevisionHashLabelKey: "10",
   561  							},
   562  						},
   563  						Status: corev1.PodStatus{
   564  							PodIP: "0.0.0.0",
   565  						},
   566  					},
   567  					master: false,
   568  					ready:  true,
   569  				},
   570  				{
   571  					pod: corev1.Pod{
   572  						ObjectMeta: metav1.ObjectMeta{
   573  							Name: "slave2",
   574  							Labels: map[string]string{
   575  								appsv1.ControllerRevisionHashLabelKey: "10",
   576  							},
   577  						},
   578  						Status: corev1.PodStatus{
   579  							PodIP: "0.0.0.1",
   580  						},
   581  					},
   582  					master: false,
   583  					ready:  true,
   584  				},
   585  				{
   586  					pod: corev1.Pod{
   587  						ObjectMeta: metav1.ObjectMeta{
   588  							Name: "master",
   589  							Labels: map[string]string{
   590  								appsv1.ControllerRevisionHashLabelKey: "10",
   591  							},
   592  						},
   593  						Status: corev1.PodStatus{
   594  							PodIP: "1.1.1.1",
   595  						},
   596  					},
   597  					master: true,
   598  					ready:  true,
   599  				},
   600  			},
   601  			ssVersion:     "1",
   602  			errExpected:   false,
   603  			bootstrapping: false,
   604  		},
   605  		{
   606  			name: "master version incorrect",
   607  			pods: []podStatus{
   608  				{
   609  					pod: corev1.Pod{
   610  						ObjectMeta: metav1.ObjectMeta{
   611  							Name: "slave1",
   612  							Labels: map[string]string{
   613  								appsv1.ControllerRevisionHashLabelKey: "10",
   614  							},
   615  						},
   616  						Status: corev1.PodStatus{
   617  							PodIP: "0.0.0.0",
   618  						},
   619  					},
   620  					master: false,
   621  					ready:  true,
   622  				},
   623  				{
   624  					pod: corev1.Pod{
   625  						ObjectMeta: metav1.ObjectMeta{
   626  							Name: "slave2",
   627  							Labels: map[string]string{
   628  								appsv1.ControllerRevisionHashLabelKey: "10",
   629  							},
   630  						},
   631  						Status: corev1.PodStatus{
   632  							PodIP: "0.0.0.1",
   633  						},
   634  					},
   635  					master: false,
   636  					ready:  true,
   637  				},
   638  				{
   639  					pod: corev1.Pod{
   640  						ObjectMeta: metav1.ObjectMeta{
   641  							Name: "master",
   642  							Labels: map[string]string{
   643  								appsv1.ControllerRevisionHashLabelKey: "1",
   644  							},
   645  						},
   646  						Status: corev1.PodStatus{
   647  							PodIP: "1.1.1.1",
   648  						},
   649  					},
   650  					master: true,
   651  					ready:  true,
   652  				},
   653  			},
   654  			ssVersion:     "10",
   655  			errExpected:   false,
   656  			bootstrapping: false,
   657  		},
   658  		{
   659  			name: "all ok, no change needed when in bootstrap mode",
   660  			pods: []podStatus{
   661  				{
   662  					pod: corev1.Pod{
   663  						ObjectMeta: metav1.ObjectMeta{
   664  							Name: "slave1",
   665  							Labels: map[string]string{
   666  								appsv1.ControllerRevisionHashLabelKey: "10",
   667  							},
   668  						},
   669  						Status: corev1.PodStatus{
   670  							PodIP: "0.0.0.0",
   671  						},
   672  					},
   673  					master: false,
   674  					ready:  true,
   675  				},
   676  				{
   677  					pod: corev1.Pod{
   678  						ObjectMeta: metav1.ObjectMeta{
   679  							Name: "slave2",
   680  							Labels: map[string]string{
   681  								appsv1.ControllerRevisionHashLabelKey: "10",
   682  							},
   683  						},
   684  						Status: corev1.PodStatus{
   685  							PodIP: "0.0.0.1",
   686  						},
   687  					},
   688  					master: false,
   689  					ready:  true,
   690  				},
   691  				{
   692  					pod: corev1.Pod{
   693  						ObjectMeta: metav1.ObjectMeta{
   694  							Name: "slave3",
   695  							Labels: map[string]string{
   696  								appsv1.ControllerRevisionHashLabelKey: "10",
   697  							},
   698  						},
   699  						Status: corev1.PodStatus{
   700  							PodIP: "1.1.1.1",
   701  						},
   702  					},
   703  					master: false,
   704  					ready:  true,
   705  				},
   706  			},
   707  			ssVersion:     "10",
   708  			errExpected:   false,
   709  			bootstrapping: true,
   710  		},
   711  		{
   712  			name: "syncing when in bootstrap mode",
   713  			pods: []podStatus{
   714  				{
   715  					pod: corev1.Pod{
   716  						ObjectMeta: metav1.ObjectMeta{
   717  							Name: "slave1",
   718  							Labels: map[string]string{
   719  								appsv1.ControllerRevisionHashLabelKey: "10",
   720  							},
   721  						},
   722  						Status: corev1.PodStatus{
   723  							PodIP: "0.0.0.0",
   724  						},
   725  					},
   726  					master: false,
   727  					ready:  true,
   728  				},
   729  				{
   730  					pod: corev1.Pod{
   731  						ObjectMeta: metav1.ObjectMeta{
   732  							Name: "slave2",
   733  							Labels: map[string]string{
   734  								appsv1.ControllerRevisionHashLabelKey: "10",
   735  							},
   736  						},
   737  						Status: corev1.PodStatus{
   738  							PodIP: "0.0.0.1",
   739  						},
   740  					},
   741  					master: false,
   742  					ready:  false,
   743  				},
   744  				{
   745  					pod: corev1.Pod{
   746  						ObjectMeta: metav1.ObjectMeta{
   747  							Name: "slave3",
   748  							Labels: map[string]string{
   749  								appsv1.ControllerRevisionHashLabelKey: "10",
   750  							},
   751  						},
   752  						Status: corev1.PodStatus{
   753  							PodIP: "1.1.1.1",
   754  						},
   755  					},
   756  					master: false,
   757  					ready:  true,
   758  				},
   759  			},
   760  			ssVersion:     "10",
   761  			errExpected:   false,
   762  			bootstrapping: true,
   763  		},
   764  		{
   765  			name: "pod version incorrect when in bootstrap mode",
   766  			pods: []podStatus{
   767  				{
   768  					pod: corev1.Pod{
   769  						ObjectMeta: metav1.ObjectMeta{
   770  							Name: "slave1",
   771  							Labels: map[string]string{
   772  								appsv1.ControllerRevisionHashLabelKey: "10",
   773  							},
   774  						},
   775  						Status: corev1.PodStatus{
   776  							PodIP: "0.0.0.0",
   777  						},
   778  					},
   779  					master: false,
   780  					ready:  true,
   781  				},
   782  				{
   783  					pod: corev1.Pod{
   784  						ObjectMeta: metav1.ObjectMeta{
   785  							Name: "slave2",
   786  							Labels: map[string]string{
   787  								appsv1.ControllerRevisionHashLabelKey: "10",
   788  							},
   789  						},
   790  						Status: corev1.PodStatus{
   791  							PodIP: "0.0.0.1",
   792  						},
   793  					},
   794  					master: false,
   795  					ready:  true,
   796  				},
   797  				{
   798  					pod: corev1.Pod{
   799  						ObjectMeta: metav1.ObjectMeta{
   800  							Name: "slave3",
   801  							Labels: map[string]string{
   802  								appsv1.ControllerRevisionHashLabelKey: "10",
   803  							},
   804  						},
   805  						Status: corev1.PodStatus{
   806  							PodIP: "1.1.1.1",
   807  						},
   808  					},
   809  					master: false,
   810  					ready:  true,
   811  				},
   812  			},
   813  			ssVersion:     "1",
   814  			errExpected:   false,
   815  			bootstrapping: true,
   816  		},
   817  		{
   818  			name: "when no master exists",
   819  			pods: []podStatus{
   820  				{
   821  					pod: corev1.Pod{
   822  						ObjectMeta: metav1.ObjectMeta{
   823  							Name: "slave1",
   824  							Labels: map[string]string{
   825  								appsv1.ControllerRevisionHashLabelKey: "10",
   826  							},
   827  						},
   828  						Status: corev1.PodStatus{
   829  							PodIP: "0.0.0.0",
   830  						},
   831  					},
   832  					master: false,
   833  					ready:  true,
   834  				},
   835  				{
   836  					pod: corev1.Pod{
   837  						ObjectMeta: metav1.ObjectMeta{
   838  							Name: "slave2",
   839  							Labels: map[string]string{
   840  								appsv1.ControllerRevisionHashLabelKey: "10",
   841  							},
   842  						},
   843  						Status: corev1.PodStatus{
   844  							PodIP: "0.0.0.1",
   845  						},
   846  					},
   847  					master: false,
   848  					ready:  true,
   849  				},
   850  				{
   851  					pod: corev1.Pod{
   852  						ObjectMeta: metav1.ObjectMeta{
   853  							Name: "slave3",
   854  							Labels: map[string]string{
   855  								appsv1.ControllerRevisionHashLabelKey: "10",
   856  							},
   857  						},
   858  						Status: corev1.PodStatus{
   859  							PodIP: "1.1.1.1",
   860  						},
   861  					},
   862  					master: false,
   863  					ready:  true,
   864  				},
   865  			},
   866  			ssVersion:     "10",
   867  			errExpected:   true,
   868  			bootstrapping: false,
   869  			noMaster:      true,
   870  		},
   871  	}
   872  	for _, test := range tests {
   873  		t.Run(test.name, func(t *testing.T) {
   874  			assert := assert.New(t)
   875  
   876  			rf := generateRF(false, test.bootstrapping)
   877  
   878  			config := generateConfig()
   879  			mrfs := &mRFService.RedisFailoverClient{}
   880  
   881  			mrfc := &mRFService.RedisFailoverCheck{}
   882  			mrfc.On("GetRedisesIPs", rf).Once().Return([]string{"0.0.0.0", "0.0.0.1", "1.1.1.1"}, nil)
   883  
   884  			next := true
   885  			if !test.bootstrapping {
   886  				master := "1.1.1.1"
   887  				if test.noMaster {
   888  					master = ""
   889  				}
   890  				mrfc.On("GetMasterIP", rf).Once().Return(master, nil)
   891  			}
   892  
   893  			for _, pod := range test.pods {
   894  				if !pod.master {
   895  					mrfc.On("CheckRedisSlavesReady", pod.pod.Status.PodIP, rf).Once().Return(pod.ready, nil)
   896  				}
   897  				if !pod.ready {
   898  					next = false
   899  					break
   900  				}
   901  			}
   902  			mrfh := &mRFService.RedisFailoverHeal{}
   903  
   904  			if next {
   905  				replicas := []string{"slave1", "slave2"}
   906  				if test.bootstrapping || test.noMaster {
   907  					replicas = append(replicas, "slave3")
   908  				}
   909  				mrfc.On("GetStatefulSetUpdateRevision", rf).Once().Return(test.ssVersion, nil)
   910  				mrfc.On("GetRedisesSlavesPods", rf).Once().Return(replicas, nil)
   911  
   912  				for _, pod := range test.pods {
   913  					mrfc.On("GetRedisRevisionHash", pod.pod.ObjectMeta.Name, rf).Once().Return(pod.pod.ObjectMeta.Labels[appsv1.ControllerRevisionHashLabelKey], nil)
   914  					if pod.pod.ObjectMeta.Labels[appsv1.ControllerRevisionHashLabelKey] != test.ssVersion {
   915  						mrfh.On("DeletePod", pod.pod.ObjectMeta.Name, rf).Once().Return(nil)
   916  						if pod.master == false {
   917  							next = false
   918  							break
   919  						}
   920  					}
   921  				}
   922  				fmt.Printf("%v - %v\n", test.name, next)
   923  				if next && !test.bootstrapping {
   924  					if test.noMaster {
   925  						mrfc.On("GetRedisesMasterPod", rf).Once().Return("", errors.New(""))
   926  					} else {
   927  						mrfc.On("GetRedisesMasterPod", rf).Once().Return("master", nil)
   928  					}
   929  				}
   930  			}
   931  
   932  			mk := &mK8SService.Services{}
   933  
   934  			handler := rfOperator.NewRedisFailoverHandler(config, mrfs, mrfc, mrfh, mk, metrics.Dummy, log.Dummy)
   935  			err := handler.UpdateRedisesPods(rf)
   936  
   937  			if test.errExpected {
   938  				assert.Error(err)
   939  			} else {
   940  				assert.NoError(err)
   941  			}
   942  
   943  			mrfc.AssertExpectations(t)
   944  			mrfh.AssertExpectations(t)
   945  
   946  		})
   947  	}
   948  }