github.com/spotahome/redis-operator@v1.2.4/operator/redisfailover/service/heal_test.go (about)

     1  package service_test
     2  
     3  import (
     4  	"errors"
     5  	"testing"
     6  	"time"
     7  
     8  	"github.com/stretchr/testify/mock"
     9  
    10  	"github.com/stretchr/testify/assert"
    11  	corev1 "k8s.io/api/core/v1"
    12  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    13  
    14  	"github.com/spotahome/redis-operator/log"
    15  	mK8SService "github.com/spotahome/redis-operator/mocks/service/k8s"
    16  	mRedisService "github.com/spotahome/redis-operator/mocks/service/redis"
    17  	rfservice "github.com/spotahome/redis-operator/operator/redisfailover/service"
    18  )
    19  
    20  func TestSetOldestAsMasterNewMasterError(t *testing.T) {
    21  	assert := assert.New(t)
    22  
    23  	rf := generateRF()
    24  
    25  	pods := &corev1.PodList{
    26  		Items: []corev1.Pod{
    27  			{
    28  				Status: corev1.PodStatus{
    29  					PodIP: "0.0.0.0",
    30  				},
    31  			},
    32  		},
    33  	}
    34  
    35  	ms := &mK8SService.Services{}
    36  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
    37  	ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil)
    38  	mr := &mRedisService.Client{}
    39  	mr.On("MakeMaster", "0.0.0.0", "0", "").Once().Return(errors.New(""))
    40  
    41  	healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{})
    42  
    43  	err := healer.SetOldestAsMaster(rf)
    44  	assert.Error(err)
    45  }
    46  
    47  func TestSetOldestAsMaster(t *testing.T) {
    48  	assert := assert.New(t)
    49  
    50  	rf := generateRF()
    51  
    52  	pods := &corev1.PodList{
    53  		Items: []corev1.Pod{
    54  			{
    55  				Status: corev1.PodStatus{
    56  					PodIP: "0.0.0.0",
    57  				},
    58  			},
    59  		},
    60  	}
    61  
    62  	ms := &mK8SService.Services{}
    63  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
    64  	ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil)
    65  	mr := &mRedisService.Client{}
    66  	mr.On("MakeMaster", "0.0.0.0", "0", "").Once().Return(nil)
    67  
    68  	healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{})
    69  
    70  	err := healer.SetOldestAsMaster(rf)
    71  	assert.NoError(err)
    72  }
    73  
    74  func TestSetOldestAsMasterMultiplePodsMakeSlaveOfError(t *testing.T) {
    75  	assert := assert.New(t)
    76  
    77  	rf := generateRF()
    78  
    79  	pods := &corev1.PodList{
    80  		Items: []corev1.Pod{
    81  			{
    82  				Status: corev1.PodStatus{
    83  					PodIP: "0.0.0.0",
    84  				},
    85  			},
    86  			{
    87  				Status: corev1.PodStatus{
    88  					PodIP: "1.1.1.1",
    89  				},
    90  			},
    91  		},
    92  	}
    93  
    94  	ms := &mK8SService.Services{}
    95  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
    96  	ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil)
    97  	mr := &mRedisService.Client{}
    98  	mr.On("MakeMaster", "0.0.0.0", "0", "").Once().Return(nil)
    99  	mr.On("MakeSlaveOfWithPort", "1.1.1.1", "0.0.0.0", "0", "").Once().Return(errors.New(""))
   100  
   101  	healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{})
   102  
   103  	err := healer.SetOldestAsMaster(rf)
   104  	assert.NoError(err)
   105  }
   106  
   107  func TestSetOldestAsMasterMultiplePods(t *testing.T) {
   108  	assert := assert.New(t)
   109  
   110  	rf := generateRF()
   111  
   112  	pods := &corev1.PodList{
   113  		Items: []corev1.Pod{
   114  			{
   115  				Status: corev1.PodStatus{
   116  					PodIP: "0.0.0.0",
   117  				},
   118  			},
   119  			{
   120  				Status: corev1.PodStatus{
   121  					PodIP: "1.1.1.1",
   122  				},
   123  			},
   124  		},
   125  	}
   126  
   127  	ms := &mK8SService.Services{}
   128  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   129  	ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil)
   130  	mr := &mRedisService.Client{}
   131  	mr.On("MakeMaster", "0.0.0.0", "0", "").Once().Return(nil)
   132  	mr.On("MakeSlaveOfWithPort", "1.1.1.1", "0.0.0.0", "0", "").Once().Return(nil)
   133  
   134  	healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{})
   135  
   136  	err := healer.SetOldestAsMaster(rf)
   137  	assert.NoError(err)
   138  }
   139  
   140  func TestSetOldestAsMasterOrdering(t *testing.T) {
   141  	assert := assert.New(t)
   142  
   143  	rf := generateRF()
   144  
   145  	pods := &corev1.PodList{
   146  		Items: []corev1.Pod{
   147  			{
   148  				ObjectMeta: metav1.ObjectMeta{
   149  					CreationTimestamp: metav1.Time{
   150  						Time: time.Now(),
   151  					},
   152  				},
   153  				Status: corev1.PodStatus{
   154  					PodIP: "0.0.0.0",
   155  				},
   156  			},
   157  			{
   158  				ObjectMeta: metav1.ObjectMeta{
   159  					CreationTimestamp: metav1.Time{
   160  						Time: time.Now().Add(-1 * time.Hour), // This is older by 1 hour
   161  					},
   162  				},
   163  				Status: corev1.PodStatus{
   164  					PodIP: "1.1.1.1",
   165  				},
   166  			},
   167  		},
   168  	}
   169  
   170  	ms := &mK8SService.Services{}
   171  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   172  	ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil)
   173  	mr := &mRedisService.Client{}
   174  	mr.On("MakeMaster", "1.1.1.1", "0", "").Once().Return(nil)
   175  	mr.On("MakeSlaveOfWithPort", "0.0.0.0", "1.1.1.1", "0", "").Once().Return(nil)
   176  
   177  	healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{})
   178  
   179  	err := healer.SetOldestAsMaster(rf)
   180  	assert.NoError(err)
   181  }
   182  
   183  func TestSetMasterOnAllMakeMasterError(t *testing.T) {
   184  	assert := assert.New(t)
   185  
   186  	rf := generateRF()
   187  
   188  	pods := &corev1.PodList{
   189  		Items: []corev1.Pod{
   190  			{
   191  				Status: corev1.PodStatus{
   192  					PodIP: "0.0.0.0",
   193  				},
   194  			},
   195  			{
   196  				Status: corev1.PodStatus{
   197  					PodIP: "1.1.1.1",
   198  				},
   199  			},
   200  		},
   201  	}
   202  
   203  	ms := &mK8SService.Services{}
   204  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   205  	ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil)
   206  	mr := &mRedisService.Client{}
   207  	mr.On("IsMaster", "0.0.0.0", "0", "").Return(false, errors.New(""))
   208  	healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{})
   209  
   210  	err := healer.SetMasterOnAll("0.0.0.0", rf)
   211  	assert.Error(err)
   212  }
   213  
   214  func TestSetMasterOnAllMakeSlaveOfError(t *testing.T) {
   215  	assert := assert.New(t)
   216  
   217  	rf := generateRF()
   218  
   219  	pods := &corev1.PodList{
   220  		Items: []corev1.Pod{
   221  			{
   222  				Status: corev1.PodStatus{
   223  					PodIP: "0.0.0.0",
   224  				},
   225  			},
   226  			{
   227  				Status: corev1.PodStatus{
   228  					PodIP: "1.1.1.1",
   229  				},
   230  			},
   231  		},
   232  	}
   233  
   234  	ms := &mK8SService.Services{}
   235  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   236  	ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil)
   237  	mr := &mRedisService.Client{}
   238  	mr.On("IsMaster", "0.0.0.0", "0", "").Return(true, nil)
   239  	mr.On("MakeSlaveOfWithPort", "1.1.1.1", "0.0.0.0", "0", "").Once().Return(errors.New(""))
   240  
   241  	healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{})
   242  
   243  	err := healer.SetMasterOnAll("0.0.0.0", rf)
   244  	assert.Error(err)
   245  }
   246  
   247  func TestSetMasterOnAll(t *testing.T) {
   248  	assert := assert.New(t)
   249  
   250  	rf := generateRF()
   251  
   252  	pods := &corev1.PodList{
   253  		Items: []corev1.Pod{
   254  			{
   255  				Status: corev1.PodStatus{
   256  					PodIP: "0.0.0.0",
   257  				},
   258  			},
   259  			{
   260  				Status: corev1.PodStatus{
   261  					PodIP: "1.1.1.1",
   262  				},
   263  			},
   264  		},
   265  	}
   266  
   267  	ms := &mK8SService.Services{}
   268  	ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   269  	ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil)
   270  	mr := &mRedisService.Client{}
   271  	mr.On("IsMaster", "0.0.0.0", "0", "").Return(true, nil)
   272  	mr.On("MakeSlaveOfWithPort", "1.1.1.1", "0.0.0.0", "0", "").Once().Return(nil)
   273  
   274  	healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{})
   275  
   276  	err := healer.SetMasterOnAll("0.0.0.0", rf)
   277  	assert.NoError(err)
   278  }
   279  
   280  func TestSetExternalMasterOnAll(t *testing.T) {
   281  	tests := []struct {
   282  		name                  string
   283  		errorOnGetStatefulSet bool
   284  		errorOnMakeSlaveOf    bool
   285  	}{
   286  		{
   287  			name: "makes all redis pods a slave of provided ip and port",
   288  		},
   289  		{
   290  			name:                  "errors on failure to get stateful set pods",
   291  			errorOnGetStatefulSet: true,
   292  		},
   293  		{
   294  			name:               "errors on failure to make pod a slave",
   295  			errorOnMakeSlaveOf: true,
   296  		},
   297  	}
   298  
   299  	for _, test := range tests {
   300  		t.Run(test.name, func(t *testing.T) {
   301  			assert := assert.New(t)
   302  			rf := generateRF()
   303  			pods := &corev1.PodList{
   304  				Items: []corev1.Pod{
   305  					{
   306  						Status: corev1.PodStatus{
   307  							PodIP: "0.0.0.0",
   308  						},
   309  					},
   310  					{
   311  						Status: corev1.PodStatus{
   312  							PodIP: "1.1.1.1",
   313  						},
   314  					},
   315  				},
   316  			}
   317  
   318  			ms := &mK8SService.Services{}
   319  			expectError := false
   320  
   321  			if test.errorOnGetStatefulSet {
   322  				expectError = true
   323  				ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(nil, errors.New(""))
   324  			} else {
   325  				ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil)
   326  			}
   327  
   328  			mr := &mRedisService.Client{}
   329  			if !expectError {
   330  				mr.On("MakeSlaveOfWithPort", "0.0.0.0", "5.5.5.5", "6379", "").Once().Return(nil)
   331  				if test.errorOnMakeSlaveOf {
   332  					expectError = true
   333  					mr.On("MakeSlaveOfWithPort", "1.1.1.1", "5.5.5.5", "6379", "").Once().Return(errors.New(""))
   334  				} else {
   335  					mr.On("MakeSlaveOfWithPort", "1.1.1.1", "5.5.5.5", "6379", "").Once().Return(nil)
   336  				}
   337  			}
   338  
   339  			healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{})
   340  
   341  			err := healer.SetExternalMasterOnAll("5.5.5.5", "6379", rf)
   342  
   343  			if expectError {
   344  				assert.Error(err)
   345  			} else {
   346  				assert.NoError(err)
   347  			}
   348  			ms.AssertExpectations(t)
   349  			mr.AssertExpectations(t)
   350  		})
   351  	}
   352  }
   353  
   354  func TestNewSentinelMonitor(t *testing.T) {
   355  	tests := []struct {
   356  		name                string
   357  		errorOnMonitorRedis bool
   358  	}{
   359  		{
   360  			name: "updates provided IP to monitor new redis master",
   361  		},
   362  		{
   363  			name:                "errors on failurer to set monitor",
   364  			errorOnMonitorRedis: true,
   365  		},
   366  	}
   367  
   368  	for _, test := range tests {
   369  		t.Run(test.name, func(t *testing.T) {
   370  			assert := assert.New(t)
   371  			rf := generateRF()
   372  			ms := &mK8SService.Services{}
   373  			mr := &mRedisService.Client{}
   374  			errorExpected := false
   375  
   376  			if test.errorOnMonitorRedis {
   377  				errorExpected = true
   378  				mr.On("MonitorRedisWithPort", "0.0.0.0", "1.1.1.1", "0", "2", "").Once().Return(errors.New(""))
   379  			} else {
   380  				mr.On("MonitorRedisWithPort", "0.0.0.0", "1.1.1.1", "0", "2", "").Once().Return(nil)
   381  			}
   382  
   383  			healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{})
   384  
   385  			err := healer.NewSentinelMonitor("0.0.0.0", "1.1.1.1", rf)
   386  
   387  			if errorExpected {
   388  				assert.Error(err)
   389  			} else {
   390  				assert.NoError(err)
   391  			}
   392  			ms.AssertExpectations(t)
   393  			mr.AssertExpectations(t)
   394  		})
   395  	}
   396  }
   397  
   398  func TestNewSentinelMonitorWithPort(t *testing.T) {
   399  	tests := []struct {
   400  		name                string
   401  		errorOnMonitorRedis bool
   402  	}{
   403  		{
   404  			name: "updates provided IP to monitor new redis master",
   405  		},
   406  		{
   407  			name:                "errors on failurer to set monitor",
   408  			errorOnMonitorRedis: true,
   409  		},
   410  	}
   411  
   412  	for _, test := range tests {
   413  		t.Run(test.name, func(t *testing.T) {
   414  			assert := assert.New(t)
   415  			rf := generateRF()
   416  			ms := &mK8SService.Services{}
   417  			mr := &mRedisService.Client{}
   418  			errorExpected := false
   419  
   420  			if test.errorOnMonitorRedis {
   421  				errorExpected = true
   422  				mr.On("MonitorRedisWithPort", "0.0.0.0", "1.1.1.1", "6379", "2", "").Once().Return(errors.New(""))
   423  			} else {
   424  				mr.On("MonitorRedisWithPort", "0.0.0.0", "1.1.1.1", "6379", "2", "").Once().Return(nil)
   425  			}
   426  
   427  			healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{})
   428  
   429  			err := healer.NewSentinelMonitorWithPort("0.0.0.0", "1.1.1.1", "6379", rf)
   430  
   431  			if errorExpected {
   432  				assert.Error(err)
   433  			} else {
   434  				assert.NoError(err)
   435  			}
   436  			ms.AssertExpectations(t)
   437  			mr.AssertExpectations(t)
   438  		})
   439  	}
   440  }