github.com/spotahome/redis-operator@v1.2.4/operator/redisfailover/service/check_test.go (about)

     1  package service_test
     2  
     3  import (
     4  	"errors"
     5  	"testing"
     6  	"time"
     7  
     8  	"github.com/stretchr/testify/mock"
     9  
    10  	"github.com/stretchr/testify/assert"
    11  	appsv1 "k8s.io/api/apps/v1"
    12  	corev1 "k8s.io/api/core/v1"
    13  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    14  
    15  	redisfailoverv1 "github.com/spotahome/redis-operator/api/redisfailover/v1"
    16  	"github.com/spotahome/redis-operator/log"
    17  	"github.com/spotahome/redis-operator/metrics"
    18  	mK8SService "github.com/spotahome/redis-operator/mocks/service/k8s"
    19  	mRedisService "github.com/spotahome/redis-operator/mocks/service/redis"
    20  	rfservice "github.com/spotahome/redis-operator/operator/redisfailover/service"
    21  )
    22  
    23  func generateRF() *redisfailoverv1.RedisFailover {
    24  	return &redisfailoverv1.RedisFailover{
    25  		ObjectMeta: metav1.ObjectMeta{
    26  			Name:      name,
    27  			Namespace: namespace,
    28  		},
    29  		Spec: redisfailoverv1.RedisFailoverSpec{
    30  			Redis: redisfailoverv1.RedisSettings{
    31  				Replicas: int32(3),
    32  			},
    33  			Sentinel: redisfailoverv1.SentinelSettings{
    34  				Replicas: int32(3),
    35  			},
    36  		},
    37  	}
    38  }
    39  
    40  func TestCheckRedisNumberError(t *testing.T) {
    41  	assert := assert.New(t)
    42  
    43  	rf := generateRF()
    44  
    45  	ms := &mK8SService.Services{}
    46  	ms.On("GetStatefulSet", namespace, rfservice.GetRedisName(rf)).Once().Return(nil, errors.New(""))
    47  	mr := &mRedisService.Client{}
    48  
    49  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
    50  
    51  	err := checker.CheckRedisNumber(rf)
    52  	assert.Error(err)
    53  }
    54  
    55  func TestCheckRedisNumberFalse(t *testing.T) {
    56  	assert := assert.New(t)
    57  
    58  	rf := generateRF()
    59  
    60  	wrongNumber := int32(4)
    61  	ss := &appsv1.StatefulSet{
    62  		Spec: appsv1.StatefulSetSpec{
    63  			Replicas: &wrongNumber,
    64  		},
    65  	}
    66  	ms := &mK8SService.Services{}
    67  	ms.On("GetStatefulSet", namespace, rfservice.GetRedisName(rf)).Once().Return(ss, nil)
    68  	mr := &mRedisService.Client{}
    69  
    70  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
    71  
    72  	err := checker.CheckRedisNumber(rf)
    73  	assert.Error(err)
    74  }
    75  
    76  func TestCheckRedisNumberTrue(t *testing.T) {
    77  	assert := assert.New(t)
    78  
    79  	rf := generateRF()
    80  
    81  	goodNumber := int32(3)
    82  	ss := &appsv1.StatefulSet{
    83  		Spec: appsv1.StatefulSetSpec{
    84  			Replicas: &goodNumber,
    85  		},
    86  	}
    87  	ms := &mK8SService.Services{}
    88  	ms.On("GetStatefulSet", namespace, rfservice.GetRedisName(rf)).Once().Return(ss, nil)
    89  	mr := &mRedisService.Client{}
    90  
    91  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
    92  
    93  	err := checker.CheckRedisNumber(rf)
    94  	assert.NoError(err)
    95  }
    96  
    97  func TestCheckSentinelNumberError(t *testing.T) {
    98  	assert := assert.New(t)
    99  
   100  	rf := generateRF()
   101  
   102  	ms := &mK8SService.Services{}
   103  	ms.On("GetDeployment", namespace, rfservice.GetSentinelName(rf)).Once().Return(nil, errors.New(""))
   104  	mr := &mRedisService.Client{}
   105  
   106  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   107  
   108  	err := checker.CheckSentinelNumber(rf)
   109  	assert.Error(err)
   110  }
   111  
   112  func TestCheckSentinelNumberFalse(t *testing.T) {
   113  	assert := assert.New(t)
   114  
   115  	rf := generateRF()
   116  
   117  	wrongNumber := int32(4)
   118  	ss := &appsv1.Deployment{
   119  		Spec: appsv1.DeploymentSpec{
   120  			Replicas: &wrongNumber,
   121  		},
   122  	}
   123  	ms := &mK8SService.Services{}
   124  	ms.On("GetDeployment", namespace, rfservice.GetSentinelName(rf)).Once().Return(ss, nil)
   125  	mr := &mRedisService.Client{}
   126  
   127  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   128  
   129  	err := checker.CheckSentinelNumber(rf)
   130  	assert.Error(err)
   131  }
   132  
   133  func TestCheckSentinelNumberTrue(t *testing.T) {
   134  	assert := assert.New(t)
   135  
   136  	rf := generateRF()
   137  
   138  	goodNumber := int32(3)
   139  	ss := &appsv1.Deployment{
   140  		Spec: appsv1.DeploymentSpec{
   141  			Replicas: &goodNumber,
   142  		},
   143  	}
   144  	ms := &mK8SService.Services{}
   145  	ms.On("GetDeployment", namespace, rfservice.GetSentinelName(rf)).Once().Return(ss, nil)
   146  	mr := &mRedisService.Client{}
   147  
   148  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   149  
   150  	err := checker.CheckSentinelNumber(rf)
   151  	assert.NoError(err)
   152  }
   153  
   154  func TestCheckAllSlavesFromMasterGetStatefulSetError(t *testing.T) {
   155  	assert := assert.New(t)
   156  
   157  	rf := generateRF()
   158  
   159  	ms := &mK8SService.Services{}
   160  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(nil, errors.New(""))
   161  	ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil)
   162  	mr := &mRedisService.Client{}
   163  
   164  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   165  
   166  	err := checker.CheckAllSlavesFromMaster("", rf)
   167  	assert.Error(err)
   168  }
   169  
   170  func TestCheckAllSlavesFromMasterGetSlaveOfError(t *testing.T) {
   171  	assert := assert.New(t)
   172  
   173  	rf := generateRF()
   174  
   175  	pods := &corev1.PodList{
   176  		Items: []corev1.Pod{
   177  			{
   178  				Status: corev1.PodStatus{
   179  					PodIP: "",
   180  					Phase: corev1.PodRunning,
   181  				},
   182  			},
   183  		},
   184  	}
   185  
   186  	ms := &mK8SService.Services{}
   187  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   188  	ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil)
   189  	mr := &mRedisService.Client{}
   190  	mr.On("GetSlaveOf", "", "0", "").Once().Return("", errors.New(""))
   191  
   192  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   193  
   194  	err := checker.CheckAllSlavesFromMaster("", rf)
   195  	assert.Error(err)
   196  }
   197  
   198  func TestCheckAllSlavesFromMasterDifferentMaster(t *testing.T) {
   199  	assert := assert.New(t)
   200  
   201  	rf := generateRF()
   202  
   203  	pods := &corev1.PodList{
   204  		Items: []corev1.Pod{
   205  			{
   206  				Status: corev1.PodStatus{
   207  					PodIP: "0.0.0.0",
   208  					Phase: corev1.PodRunning,
   209  				},
   210  			},
   211  		},
   212  	}
   213  
   214  	ms := &mK8SService.Services{}
   215  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   216  	ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil)
   217  	mr := &mRedisService.Client{}
   218  	mr.On("GetSlaveOf", "0.0.0.0", "0", "").Once().Return("1.1.1.1", nil)
   219  
   220  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   221  
   222  	err := checker.CheckAllSlavesFromMaster("0.0.0.0", rf)
   223  	assert.Error(err)
   224  }
   225  
   226  func TestCheckAllSlavesFromMaster(t *testing.T) {
   227  	assert := assert.New(t)
   228  
   229  	rf := generateRF()
   230  
   231  	pods := &corev1.PodList{
   232  		Items: []corev1.Pod{
   233  			{
   234  				Status: corev1.PodStatus{
   235  					PodIP: "0.0.0.0",
   236  					Phase: corev1.PodRunning,
   237  				},
   238  			},
   239  		},
   240  	}
   241  
   242  	ms := &mK8SService.Services{}
   243  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   244  	ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil)
   245  	mr := &mRedisService.Client{}
   246  	mr.On("GetSlaveOf", "0.0.0.0", "0", "").Once().Return("1.1.1.1", nil)
   247  
   248  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   249  
   250  	err := checker.CheckAllSlavesFromMaster("1.1.1.1", rf)
   251  	assert.NoError(err)
   252  }
   253  
   254  func TestCheckSentinelNumberInMemoryGetDeploymentPodsError(t *testing.T) {
   255  	assert := assert.New(t)
   256  
   257  	rf := generateRF()
   258  
   259  	ms := &mK8SService.Services{}
   260  	mr := &mRedisService.Client{}
   261  	mr.On("GetNumberSentinelsInMemory", "1.1.1.1").Once().Return(int32(0), errors.New("expected error"))
   262  
   263  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   264  
   265  	err := checker.CheckSentinelNumberInMemory("1.1.1.1", rf)
   266  	assert.Error(err)
   267  }
   268  
   269  func TestCheckSentinelNumberInMemoryGetNumberSentinelInMemoryError(t *testing.T) {
   270  	assert := assert.New(t)
   271  
   272  	rf := generateRF()
   273  
   274  	ms := &mK8SService.Services{}
   275  	mr := &mRedisService.Client{}
   276  	mr.On("GetNumberSentinelsInMemory", "1.1.1.1").Once().Return(int32(0), errors.New(""))
   277  
   278  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   279  
   280  	err := checker.CheckSentinelNumberInMemory("1.1.1.1", rf)
   281  	assert.Error(err)
   282  }
   283  
   284  func TestCheckSentinelNumberInMemoryNumberMismatch(t *testing.T) {
   285  	assert := assert.New(t)
   286  
   287  	rf := generateRF()
   288  
   289  	ms := &mK8SService.Services{}
   290  	mr := &mRedisService.Client{}
   291  	mr.On("GetNumberSentinelsInMemory", "1.1.1.1").Once().Return(int32(4), nil)
   292  
   293  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   294  
   295  	err := checker.CheckSentinelNumberInMemory("1.1.1.1", rf)
   296  	assert.Error(err)
   297  }
   298  
   299  func TestCheckSentinelNumberInMemory(t *testing.T) {
   300  	assert := assert.New(t)
   301  
   302  	rf := generateRF()
   303  
   304  	ms := &mK8SService.Services{}
   305  	mr := &mRedisService.Client{}
   306  	mr.On("GetNumberSentinelsInMemory", "1.1.1.1").Once().Return(int32(3), nil)
   307  
   308  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   309  
   310  	err := checker.CheckSentinelNumberInMemory("1.1.1.1", rf)
   311  	assert.NoError(err)
   312  }
   313  
   314  func TestCheckSentinelSlavesNumberInMemoryGetNumberSentinelSlavesInMemoryError(t *testing.T) {
   315  	assert := assert.New(t)
   316  
   317  	rf := generateRF()
   318  
   319  	ms := &mK8SService.Services{}
   320  	mr := &mRedisService.Client{}
   321  	mr.On("GetNumberSentinelSlavesInMemory", "1.1.1.1").Once().Return(int32(0), errors.New(""))
   322  
   323  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   324  
   325  	err := checker.CheckSentinelSlavesNumberInMemory("1.1.1.1", rf)
   326  	assert.Error(err)
   327  }
   328  
   329  func TestCheckSentinelSlavesNumberInMemoryReplicasMismatch(t *testing.T) {
   330  	assert := assert.New(t)
   331  
   332  	rf := generateRF()
   333  
   334  	ms := &mK8SService.Services{}
   335  	mr := &mRedisService.Client{}
   336  	mr.On("GetNumberSentinelSlavesInMemory", "1.1.1.1").Once().Return(int32(3), nil)
   337  
   338  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   339  
   340  	err := checker.CheckSentinelSlavesNumberInMemory("1.1.1.1", rf)
   341  	assert.Error(err)
   342  }
   343  
   344  func TestCheckSentinelSlavesNumberInMemory(t *testing.T) {
   345  	assert := assert.New(t)
   346  
   347  	rf := generateRF()
   348  	rf.Spec.Redis.Replicas = 5
   349  
   350  	ms := &mK8SService.Services{}
   351  	mr := &mRedisService.Client{}
   352  	mr.On("GetNumberSentinelSlavesInMemory", "1.1.1.1").Once().Return(int32(4), nil)
   353  
   354  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   355  
   356  	err := checker.CheckSentinelSlavesNumberInMemory("1.1.1.1", rf)
   357  	assert.NoError(err)
   358  }
   359  
   360  func TestCheckSentinelMonitorGetSentinelMonitorError(t *testing.T) {
   361  	assert := assert.New(t)
   362  
   363  	ms := &mK8SService.Services{}
   364  	mr := &mRedisService.Client{}
   365  	mr.On("GetSentinelMonitor", "0.0.0.0").Once().Return("", "", errors.New(""))
   366  
   367  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   368  
   369  	err := checker.CheckSentinelMonitor("0.0.0.0", "1.1.1.1")
   370  	assert.Error(err)
   371  }
   372  
   373  func TestCheckSentinelMonitorMismatch(t *testing.T) {
   374  	assert := assert.New(t)
   375  
   376  	ms := &mK8SService.Services{}
   377  	mr := &mRedisService.Client{}
   378  	mr.On("GetSentinelMonitor", "0.0.0.0").Once().Return("2.2.2.2", "6379", nil)
   379  
   380  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   381  
   382  	err := checker.CheckSentinelMonitor("0.0.0.0", "1.1.1.1")
   383  	assert.Error(err)
   384  }
   385  
   386  func TestCheckSentinelMonitor(t *testing.T) {
   387  	assert := assert.New(t)
   388  
   389  	ms := &mK8SService.Services{}
   390  	mr := &mRedisService.Client{}
   391  	mr.On("GetSentinelMonitor", "0.0.0.0").Once().Return("1.1.1.1", "6379", nil)
   392  
   393  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   394  
   395  	err := checker.CheckSentinelMonitor("0.0.0.0", "1.1.1.1")
   396  	assert.NoError(err)
   397  }
   398  
   399  func TestCheckSentinelMonitorWithPort(t *testing.T) {
   400  	assert := assert.New(t)
   401  
   402  	ms := &mK8SService.Services{}
   403  	mr := &mRedisService.Client{}
   404  	mr.On("GetSentinelMonitor", "0.0.0.0").Once().Return("1.1.1.1", "6379", nil)
   405  
   406  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   407  
   408  	err := checker.CheckSentinelMonitor("0.0.0.0", "1.1.1.1", "6379")
   409  	assert.NoError(err)
   410  }
   411  
   412  func TestCheckSentinelMonitorWithPortMismatch(t *testing.T) {
   413  	assert := assert.New(t)
   414  
   415  	ms := &mK8SService.Services{}
   416  	mr := &mRedisService.Client{}
   417  	mr.On("GetSentinelMonitor", "0.0.0.0").Once().Return("1.1.1.1", "6379", nil)
   418  
   419  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   420  
   421  	err := checker.CheckSentinelMonitor("0.0.0.0", "0.0.0.0", "6379")
   422  	assert.Error(err)
   423  }
   424  
   425  func TestCheckSentinelMonitorWithPortIPMismatch(t *testing.T) {
   426  	assert := assert.New(t)
   427  
   428  	ms := &mK8SService.Services{}
   429  	mr := &mRedisService.Client{}
   430  	mr.On("GetSentinelMonitor", "0.0.0.0").Once().Return("1.1.1.1", "6379", nil)
   431  
   432  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   433  
   434  	err := checker.CheckSentinelMonitor("0.0.0.0", "1.1.1.1", "6380")
   435  	assert.Error(err)
   436  }
   437  
   438  func TestGetMasterIPGetStatefulSetPodsError(t *testing.T) {
   439  	assert := assert.New(t)
   440  
   441  	rf := generateRF()
   442  
   443  	ms := &mK8SService.Services{}
   444  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(nil, errors.New(""))
   445  	mr := &mRedisService.Client{}
   446  
   447  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   448  
   449  	_, err := checker.GetMasterIP(rf)
   450  	assert.Error(err)
   451  }
   452  
   453  func TestGetMasterIPIsMasterError(t *testing.T) {
   454  	assert := assert.New(t)
   455  
   456  	rf := generateRF()
   457  
   458  	pods := &corev1.PodList{
   459  		Items: []corev1.Pod{
   460  			{
   461  				Status: corev1.PodStatus{
   462  					PodIP: "0.0.0.0",
   463  					Phase: corev1.PodRunning,
   464  				},
   465  			},
   466  		},
   467  	}
   468  
   469  	ms := &mK8SService.Services{}
   470  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   471  	mr := &mRedisService.Client{}
   472  	mr.On("IsMaster", "0.0.0.0", "0", "").Once().Return(false, errors.New(""))
   473  
   474  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   475  
   476  	_, err := checker.GetMasterIP(rf)
   477  	assert.Error(err)
   478  }
   479  
   480  func TestGetMasterIPMultipleMastersError(t *testing.T) {
   481  	assert := assert.New(t)
   482  
   483  	rf := generateRF()
   484  
   485  	pods := &corev1.PodList{
   486  		Items: []corev1.Pod{
   487  			{
   488  				Status: corev1.PodStatus{
   489  					PodIP: "0.0.0.0",
   490  					Phase: corev1.PodRunning,
   491  				},
   492  			},
   493  			{
   494  				Status: corev1.PodStatus{
   495  					PodIP: "1.1.1.1",
   496  					Phase: corev1.PodRunning,
   497  				},
   498  			},
   499  		},
   500  	}
   501  
   502  	ms := &mK8SService.Services{}
   503  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   504  	mr := &mRedisService.Client{}
   505  	mr.On("IsMaster", "0.0.0.0", "0", "").Once().Return(true, nil)
   506  	mr.On("IsMaster", "1.1.1.1", "0", "").Once().Return(true, nil)
   507  
   508  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   509  
   510  	_, err := checker.GetMasterIP(rf)
   511  	assert.Error(err)
   512  }
   513  
   514  func TestGetMasterIP(t *testing.T) {
   515  	assert := assert.New(t)
   516  
   517  	rf := generateRF()
   518  
   519  	pods := &corev1.PodList{
   520  		Items: []corev1.Pod{
   521  			{
   522  				Status: corev1.PodStatus{
   523  					PodIP: "0.0.0.0",
   524  					Phase: corev1.PodRunning,
   525  				},
   526  			},
   527  			{
   528  				Status: corev1.PodStatus{
   529  					PodIP: "1.1.1.1",
   530  					Phase: corev1.PodRunning,
   531  				},
   532  			},
   533  		},
   534  	}
   535  
   536  	ms := &mK8SService.Services{}
   537  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   538  	mr := &mRedisService.Client{}
   539  	mr.On("IsMaster", "0.0.0.0", "0", "").Once().Return(true, nil)
   540  	mr.On("IsMaster", "1.1.1.1", "0", "").Once().Return(false, nil)
   541  
   542  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   543  
   544  	master, err := checker.GetMasterIP(rf)
   545  	assert.NoError(err)
   546  	assert.Equal("0.0.0.0", master, "the master should be the expected")
   547  }
   548  
   549  func TestGetNumberMastersGetStatefulSetPodsError(t *testing.T) {
   550  	assert := assert.New(t)
   551  
   552  	rf := generateRF()
   553  
   554  	ms := &mK8SService.Services{}
   555  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(nil, errors.New(""))
   556  	mr := &mRedisService.Client{}
   557  
   558  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   559  
   560  	_, err := checker.GetNumberMasters(rf)
   561  	assert.Error(err)
   562  }
   563  
   564  func TestGetNumberMastersIsMasterError(t *testing.T) {
   565  	assert := assert.New(t)
   566  
   567  	rf := generateRF()
   568  
   569  	pods := &corev1.PodList{
   570  		Items: []corev1.Pod{
   571  			{
   572  				Status: corev1.PodStatus{
   573  					PodIP: "0.0.0.0",
   574  					Phase: corev1.PodRunning,
   575  				},
   576  			},
   577  		},
   578  	}
   579  
   580  	ms := &mK8SService.Services{}
   581  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   582  	mr := &mRedisService.Client{}
   583  	mr.On("IsMaster", "0.0.0.0", "0", "").Once().Return(true, errors.New(""))
   584  
   585  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   586  
   587  	_, err := checker.GetNumberMasters(rf)
   588  	assert.NoError(err)
   589  }
   590  
   591  func TestGetNumberMasters(t *testing.T) {
   592  	assert := assert.New(t)
   593  
   594  	rf := generateRF()
   595  
   596  	pods := &corev1.PodList{
   597  		Items: []corev1.Pod{
   598  			{
   599  				Status: corev1.PodStatus{
   600  					PodIP: "0.0.0.0",
   601  					Phase: corev1.PodRunning,
   602  				},
   603  			},
   604  			{
   605  				Status: corev1.PodStatus{
   606  					PodIP: "1.1.1.1",
   607  					Phase: corev1.PodRunning,
   608  				},
   609  			},
   610  		},
   611  	}
   612  
   613  	ms := &mK8SService.Services{}
   614  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   615  	mr := &mRedisService.Client{}
   616  	mr.On("IsMaster", "0.0.0.0", "0", "").Once().Return(true, nil)
   617  	mr.On("IsMaster", "1.1.1.1", "0", "").Once().Return(false, nil)
   618  
   619  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   620  
   621  	masterNumber, err := checker.GetNumberMasters(rf)
   622  	assert.NoError(err)
   623  	assert.Equal(1, masterNumber, "the master number should be ok")
   624  }
   625  
   626  func TestGetNumberMastersTwo(t *testing.T) {
   627  	assert := assert.New(t)
   628  
   629  	rf := generateRF()
   630  
   631  	pods := &corev1.PodList{
   632  		Items: []corev1.Pod{
   633  			{
   634  				Status: corev1.PodStatus{
   635  					PodIP: "0.0.0.0",
   636  					Phase: corev1.PodRunning,
   637  				},
   638  			},
   639  			{
   640  				Status: corev1.PodStatus{
   641  					PodIP: "1.1.1.1",
   642  					Phase: corev1.PodRunning,
   643  				},
   644  			},
   645  		},
   646  	}
   647  
   648  	ms := &mK8SService.Services{}
   649  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   650  	mr := &mRedisService.Client{}
   651  	mr.On("IsMaster", "0.0.0.0", "0", "").Once().Return(true, nil)
   652  	mr.On("IsMaster", "1.1.1.1", "0", "").Once().Return(true, nil)
   653  
   654  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   655  
   656  	masterNumber, err := checker.GetNumberMasters(rf)
   657  	assert.NoError(err)
   658  	assert.Equal(2, masterNumber, "the master number should be ok")
   659  }
   660  
   661  func TestGetMaxRedisPodTimeGetStatefulSetPodsError(t *testing.T) {
   662  	assert := assert.New(t)
   663  
   664  	rf := generateRF()
   665  
   666  	ms := &mK8SService.Services{}
   667  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(nil, errors.New(""))
   668  	mr := &mRedisService.Client{}
   669  
   670  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   671  
   672  	_, err := checker.GetMaxRedisPodTime(rf)
   673  	assert.Error(err)
   674  }
   675  
   676  func TestGetMaxRedisPodTime(t *testing.T) {
   677  	assert := assert.New(t)
   678  
   679  	rf := generateRF()
   680  
   681  	now := time.Now()
   682  	oneHour := now.Add(-1 * time.Hour)
   683  	oneMinute := now.Add(-1 * time.Minute)
   684  
   685  	pods := &corev1.PodList{
   686  		Items: []corev1.Pod{
   687  			{
   688  				Status: corev1.PodStatus{
   689  					StartTime: &metav1.Time{
   690  						Time: oneHour,
   691  					},
   692  				},
   693  			},
   694  			{
   695  				Status: corev1.PodStatus{
   696  					StartTime: &metav1.Time{
   697  						Time: oneMinute,
   698  					},
   699  				},
   700  			},
   701  		},
   702  	}
   703  
   704  	ms := &mK8SService.Services{}
   705  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   706  	mr := &mRedisService.Client{}
   707  
   708  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   709  
   710  	maxTime, err := checker.GetMaxRedisPodTime(rf)
   711  	assert.NoError(err)
   712  
   713  	expected := now.Sub(oneHour).Round(time.Second)
   714  	assert.Equal(expected, maxTime.Round(time.Second), "the closest time should be given")
   715  }
   716  
   717  func TestGetRedisPodsNames(t *testing.T) {
   718  	assert := assert.New(t)
   719  	rf := generateRF()
   720  
   721  	pods := &corev1.PodList{
   722  		Items: []corev1.Pod{
   723  			{
   724  				ObjectMeta: metav1.ObjectMeta{
   725  					Name: "slave1",
   726  				},
   727  				Status: corev1.PodStatus{
   728  					Phase: corev1.PodRunning,
   729  					PodIP: "0.0.0.0",
   730  				},
   731  			},
   732  			{
   733  				ObjectMeta: metav1.ObjectMeta{
   734  					Name: "master",
   735  				},
   736  				Status: corev1.PodStatus{
   737  					Phase: corev1.PodRunning,
   738  					PodIP: "1.1.1.1",
   739  				},
   740  			},
   741  			{
   742  				ObjectMeta: metav1.ObjectMeta{
   743  					Name: "slave2",
   744  				},
   745  				Status: corev1.PodStatus{
   746  					Phase: corev1.PodRunning,
   747  					PodIP: "0.0.0.0",
   748  				},
   749  			},
   750  		},
   751  	}
   752  
   753  	ms := &mK8SService.Services{}
   754  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   755  	mr := &mRedisService.Client{}
   756  	mr.On("IsMaster", "0.0.0.0", "0", "").Twice().Return(false, nil)
   757  	mr.On("IsMaster", "1.1.1.1", "0", "").Once().Return(true, nil)
   758  
   759  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   760  	master, err := checker.GetRedisesMasterPod(rf)
   761  
   762  	assert.NoError(err)
   763  
   764  	assert.Equal(master, "master")
   765  
   766  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   767  	mr.On("IsMaster", "0.0.0.0", "0", "").Twice().Return(false, nil)
   768  	mr.On("IsMaster", "1.1.1.1", "0", "").Once().Return(true, nil)
   769  
   770  	namePods, err := checker.GetRedisesSlavesPods(rf)
   771  
   772  	assert.NoError(err)
   773  
   774  	assert.Equal(namePods, []string{"slave1", "slave2"})
   775  }
   776  
   777  func TestGetStatefulSetUpdateRevision(t *testing.T) {
   778  	tests := []struct {
   779  		name             string
   780  		ss               *appsv1.StatefulSet
   781  		expectedUVersion string
   782  		expectedError    error
   783  	}{
   784  		{
   785  			name: "revision ok",
   786  			ss: &appsv1.StatefulSet{
   787  				Status: appsv1.StatefulSetStatus{
   788  					UpdateRevision: "10",
   789  				},
   790  			},
   791  			expectedUVersion: "10",
   792  			expectedError:    nil,
   793  		},
   794  		{
   795  			name:             "no stateful set",
   796  			ss:               nil,
   797  			expectedUVersion: "",
   798  			expectedError:    errors.New("not found"),
   799  		},
   800  	}
   801  
   802  	for _, test := range tests {
   803  		assert := assert.New(t)
   804  
   805  		rf := generateRF()
   806  		ms := &mK8SService.Services{}
   807  		ms.On("GetStatefulSet", namespace, rfservice.GetRedisName(rf)).Once().Return(test.ss, nil)
   808  		mr := &mRedisService.Client{}
   809  
   810  		checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   811  		version, err := checker.GetStatefulSetUpdateRevision(rf)
   812  
   813  		if test.expectedError == nil {
   814  			assert.NoError(err)
   815  		} else {
   816  			assert.Error(err)
   817  		}
   818  
   819  		assert.Equal(version, test.expectedUVersion)
   820  	}
   821  
   822  }
   823  
   824  func TestGetRedisRevisionHash(t *testing.T) {
   825  	tests := []struct {
   826  		name          string
   827  		pod           *corev1.Pod
   828  		expectedHash  string
   829  		expectedError error
   830  	}{
   831  		{
   832  			name: "has ok",
   833  			pod: &corev1.Pod{
   834  				ObjectMeta: metav1.ObjectMeta{
   835  					Labels: map[string]string{
   836  						appsv1.ControllerRevisionHashLabelKey: "10",
   837  					},
   838  				},
   839  			},
   840  			expectedHash:  "10",
   841  			expectedError: nil,
   842  		},
   843  		{
   844  			name:          "no pod",
   845  			pod:           nil,
   846  			expectedHash:  "",
   847  			expectedError: errors.New("not found"),
   848  		},
   849  	}
   850  
   851  	for _, test := range tests {
   852  		assert := assert.New(t)
   853  
   854  		rf := generateRF()
   855  		ms := &mK8SService.Services{}
   856  		ms.On("GetPod", namespace, "namepod").Once().Return(test.pod, nil)
   857  		mr := &mRedisService.Client{}
   858  
   859  		checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   860  		hash, err := checker.GetRedisRevisionHash("namepod", rf)
   861  
   862  		if test.expectedError == nil {
   863  			assert.NoError(err)
   864  		} else {
   865  			assert.Error(err)
   866  		}
   867  
   868  		assert.Equal(hash, test.expectedHash)
   869  	}
   870  
   871  }
   872  
   873  func TestClusterRunning(t *testing.T) {
   874  	assert := assert.New(t)
   875  
   876  	rf := generateRF()
   877  
   878  	allRunning := &corev1.PodList{
   879  		Items: []corev1.Pod{
   880  			{
   881  				Status: corev1.PodStatus{
   882  					PodIP: "0.0.0.0",
   883  					Phase: corev1.PodRunning,
   884  				},
   885  			},
   886  			{
   887  				Status: corev1.PodStatus{
   888  					PodIP: "1.1.1.1",
   889  					Phase: corev1.PodRunning,
   890  				},
   891  			},
   892  			{
   893  				Status: corev1.PodStatus{
   894  					PodIP: "1.1.1.1",
   895  					Phase: corev1.PodRunning,
   896  				},
   897  			},
   898  		},
   899  	}
   900  
   901  	notAllRunning := &corev1.PodList{
   902  		Items: []corev1.Pod{
   903  			{
   904  				Status: corev1.PodStatus{
   905  					PodIP: "0.0.0.0",
   906  					Phase: corev1.PodRunning,
   907  				},
   908  			},
   909  			{
   910  				Status: corev1.PodStatus{
   911  					PodIP: "1.1.1.1",
   912  					Phase: corev1.PodPending,
   913  				},
   914  			},
   915  			{
   916  				Status: corev1.PodStatus{
   917  					PodIP: "1.1.1.1",
   918  					Phase: corev1.PodRunning,
   919  				},
   920  			},
   921  		},
   922  	}
   923  
   924  	notAllReplicas := &corev1.PodList{
   925  		Items: []corev1.Pod{
   926  			{
   927  				Status: corev1.PodStatus{
   928  					PodIP: "0.0.0.0",
   929  					Phase: corev1.PodRunning,
   930  				},
   931  			},
   932  			{
   933  				Status: corev1.PodStatus{
   934  					PodIP: "1.1.1.1",
   935  					Phase: corev1.PodRunning,
   936  				},
   937  			},
   938  		},
   939  	}
   940  
   941  	ms := &mK8SService.Services{}
   942  	ms.On("GetDeploymentPods", namespace, rfservice.GetSentinelName(rf)).Once().Return(allRunning, nil)
   943  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(allRunning, nil)
   944  	mr := &mRedisService.Client{}
   945  
   946  	checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}, metrics.Dummy)
   947  
   948  	assert.True(checker.IsClusterRunning(rf))
   949  
   950  	ms.On("GetDeploymentPods", namespace, rfservice.GetSentinelName(rf)).Once().Return(allRunning, nil)
   951  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(notAllReplicas, nil)
   952  	assert.False(checker.IsClusterRunning(rf))
   953  
   954  	ms.On("GetDeploymentPods", namespace, rfservice.GetSentinelName(rf)).Once().Return(notAllRunning, nil)
   955  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(allRunning, nil)
   956  	assert.False(checker.IsClusterRunning(rf))
   957  
   958  }