diff --git a/api/redisfailover/v1/defaults.go b/api/redisfailover/v1/defaults.go index a0125369d..cc8259141 100644 --- a/api/redisfailover/v1/defaults.go +++ b/api/redisfailover/v1/defaults.go @@ -6,7 +6,7 @@ const ( defaultSentinelExporterImage = "quay.io/oliver006/redis_exporter:v1.43.0" defaultExporterImage = "quay.io/oliver006/redis_exporter:v1.43.0" defaultImage = "redis:6.2.6-alpine" - defaultRedisPort = "6379" + defaultRedisPort = 6379 ) var ( diff --git a/api/redisfailover/v1/types.go b/api/redisfailover/v1/types.go index 157f036d1..8011d6714 100644 --- a/api/redisfailover/v1/types.go +++ b/api/redisfailover/v1/types.go @@ -40,6 +40,7 @@ type RedisSettings struct { Image string `json:"image,omitempty"` ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` Replicas int32 `json:"replicas,omitempty"` + Port int32 `json:"port,omitempty"` Resources corev1.ResourceRequirements `json:"resources,omitempty"` CustomConfig []string `json:"customConfig,omitempty"` CustomCommandRenames []RedisCommandRename `json:"customCommandRenames,omitempty"` diff --git a/api/redisfailover/v1/validate.go b/api/redisfailover/v1/validate.go index 3b38d54ab..ec5a7e66e 100644 --- a/api/redisfailover/v1/validate.go +++ b/api/redisfailover/v1/validate.go @@ -3,6 +3,7 @@ package v1 import ( "errors" "fmt" + "strconv" ) const ( @@ -22,7 +23,7 @@ func (r *RedisFailover) Validate() error { } if r.Spec.BootstrapNode.Port == "" { - r.Spec.BootstrapNode.Port = defaultRedisPort + r.Spec.BootstrapNode.Port = strconv.Itoa(defaultRedisPort) } initialRedisCustomConfig = bootstrappingRedisCustomConfig } @@ -41,6 +42,10 @@ func (r *RedisFailover) Validate() error { r.Spec.Redis.Replicas = defaultRedisNumber } + if r.Spec.Redis.Port <= 0 { + r.Spec.Redis.Port = defaultRedisPort + } + if r.Spec.Sentinel.Replicas <= 0 { r.Spec.Sentinel.Replicas = defaultSentinelNumber } diff --git a/api/redisfailover/v1/validate_test.go b/api/redisfailover/v1/validate_test.go index f130b59bf..0cc8302de 100644 --- a/api/redisfailover/v1/validate_test.go +++ b/api/redisfailover/v1/validate_test.go @@ -104,6 +104,7 @@ func TestValidate(t *testing.T) { Redis: RedisSettings{ Image: defaultImage, Replicas: defaultRedisNumber, + Port: defaultRedisPort, Exporter: RedisExporter{ Image: defaultExporterImage, }, diff --git a/example/redisfailover/custom-port.yaml b/example/redisfailover/custom-port.yaml new file mode 100644 index 000000000..3132091ce --- /dev/null +++ b/example/redisfailover/custom-port.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: ports +--- +apiVersion: databases.spotahome.com/v1 +kind: RedisFailover +metadata: + name: redis-ports + namespace: ports +spec: + redis: + exporter: + enabled: true + port: 12345 + replicas: 3 + sentinel: + exporter: + enabled: true + replicas: 3 diff --git a/manifests/databases.spotahome.com_redisfailovers.yaml b/manifests/databases.spotahome.com_redisfailovers.yaml index e317ad25f..9a7392455 100644 --- a/manifests/databases.spotahome.com_redisfailovers.yaml +++ b/manifests/databases.spotahome.com_redisfailovers.yaml @@ -1473,6 +1473,9 @@ spec: additionalProperties: type: string type: object + port: + format: int32 + type: integer priorityClassName: type: string replicas: diff --git a/mocks/service/redis/Client.go b/mocks/service/redis/Client.go index b8c5ae460..b262bf868 100644 --- a/mocks/service/redis/Client.go +++ b/mocks/service/redis/Client.go @@ -79,20 +79,20 @@ func (_m *Client) GetSentinelMonitor(ip string) (string, string, error) { return r0, r1, r2 } -// GetSlaveOf provides a mock function with given fields: ip, password -func (_m *Client) GetSlaveOf(ip string, password string) (string, error) { - ret := _m.Called(ip, password) +// GetSlaveOf provides a mock function with given fields: ip, port, password +func (_m *Client) GetSlaveOf(ip string, port string, password string) (string, error) { + ret := _m.Called(ip, port, password) var r0 string - if rf, ok := ret.Get(0).(func(string, string) string); ok { - r0 = rf(ip, password) + if rf, ok := ret.Get(0).(func(string, string, string) string); ok { + r0 = rf(ip, port, password) } else { r0 = ret.Get(0).(string) } var r1 error - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(ip, password) + if rf, ok := ret.Get(1).(func(string, string, string) error); ok { + r1 = rf(ip, port, password) } else { r1 = ret.Error(1) } @@ -100,20 +100,20 @@ func (_m *Client) GetSlaveOf(ip string, password string) (string, error) { return r0, r1 } -// IsMaster provides a mock function with given fields: ip, password -func (_m *Client) IsMaster(ip string, password string) (bool, error) { - ret := _m.Called(ip, password) +// IsMaster provides a mock function with given fields: ip, port, password +func (_m *Client) IsMaster(ip string, port string, password string) (bool, error) { + ret := _m.Called(ip, port, password) var r0 bool - if rf, ok := ret.Get(0).(func(string, string) bool); ok { - r0 = rf(ip, password) + if rf, ok := ret.Get(0).(func(string, string, string) bool); ok { + r0 = rf(ip, port, password) } else { r0 = ret.Get(0).(bool) } var r1 error - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(ip, password) + if rf, ok := ret.Get(1).(func(string, string, string) error); ok { + r1 = rf(ip, port, password) } else { r1 = ret.Error(1) } @@ -121,13 +121,13 @@ func (_m *Client) IsMaster(ip string, password string) (bool, error) { return r0, r1 } -// MakeMaster provides a mock function with given fields: ip, password -func (_m *Client) MakeMaster(ip string, password string) error { - ret := _m.Called(ip, password) +// MakeMaster provides a mock function with given fields: ip, port, password +func (_m *Client) MakeMaster(ip string, port string, password string) error { + ret := _m.Called(ip, port, password) var r0 error - if rf, ok := ret.Get(0).(func(string, string) error); ok { - r0 = rf(ip, password) + if rf, ok := ret.Get(0).(func(string, string, string) error); ok { + r0 = rf(ip, port, password) } else { r0 = ret.Error(0) } @@ -205,13 +205,13 @@ func (_m *Client) ResetSentinel(ip string) error { return r0 } -// SetCustomRedisConfig provides a mock function with given fields: ip, configs, password -func (_m *Client) SetCustomRedisConfig(ip string, configs []string, password string) error { - ret := _m.Called(ip, configs, password) +// SetCustomRedisConfig provides a mock function with given fields: ip, port, configs, password +func (_m *Client) SetCustomRedisConfig(ip string, port string, configs []string, password string) error { + ret := _m.Called(ip, port, configs, password) var r0 error - if rf, ok := ret.Get(0).(func(string, []string, string) error); ok { - r0 = rf(ip, configs, password) + if rf, ok := ret.Get(0).(func(string, string, []string, string) error); ok { + r0 = rf(ip, port, configs, password) } else { r0 = ret.Error(0) } @@ -233,20 +233,20 @@ func (_m *Client) SetCustomSentinelConfig(ip string, configs []string) error { return r0 } -// SlaveIsReady provides a mock function with given fields: ip, password -func (_m *Client) SlaveIsReady(ip string, password string) (bool, error) { - ret := _m.Called(ip, password) +// SlaveIsReady provides a mock function with given fields: ip, port, password +func (_m *Client) SlaveIsReady(ip string, port string, password string) (bool, error) { + ret := _m.Called(ip, port, password) var r0 bool - if rf, ok := ret.Get(0).(func(string, string) bool); ok { - r0 = rf(ip, password) + if rf, ok := ret.Get(0).(func(string, string, string) bool); ok { + r0 = rf(ip, port, password) } else { r0 = ret.Get(0).(bool) } var r1 error - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(ip, password) + if rf, ok := ret.Get(1).(func(string, string, string) error); ok { + r1 = rf(ip, port, password) } else { r1 = ret.Error(1) } diff --git a/operator/redisfailover/checker.go b/operator/redisfailover/checker.go index e54991dce..583d74628 100644 --- a/operator/redisfailover/checker.go +++ b/operator/redisfailover/checker.go @@ -2,6 +2,7 @@ package redisfailover import ( "errors" + "strconv" "time" redisfailoverv1 "github.com/spotahome/redis-operator/api/redisfailover/v1" @@ -23,9 +24,9 @@ func (r *RedisFailoverHandler) UpdateRedisesPods(rf *redisfailoverv1.RedisFailov masterIP, _ = r.rfChecker.GetMasterIP(rf) } // No perform updates when nodes are syncing, still not connected, etc. - for _, rp := range redises { - if rp != masterIP { - ready, err := r.rfChecker.CheckRedisSlavesReady(rp, rf) + for _, rip := range redises { + if rip != masterIP { + ready, err := r.rfChecker.CheckRedisSlavesReady(rip, rf) if err != nil { return err } @@ -169,8 +170,10 @@ func (r *RedisFailoverHandler) CheckAndHeal(rf *redisfailoverv1.RedisFailover) e if err != nil { return err } + + port := getRedisPort(rf.Spec.Redis.Port) for _, sip := range sentinels { - if err := r.rfChecker.CheckSentinelMonitor(sip, master); err != nil { + if err := r.rfChecker.CheckSentinelMonitor(sip, master, port); err != nil { r.logger.Debug("Sentinel is not monitoring the correct master") if err := r.rfHealer.NewSentinelMonitor(sip, master, rf); err != nil { return err @@ -260,3 +263,7 @@ func (r *RedisFailoverHandler) checkAndHealSentinels(rf *redisfailoverv1.RedisFa } return nil } + +func getRedisPort(p int32) string { + return strconv.Itoa(int(p)) +} diff --git a/operator/redisfailover/checker_test.go b/operator/redisfailover/checker_test.go index d352fbf75..02031e724 100644 --- a/operator/redisfailover/checker_test.go +++ b/operator/redisfailover/checker_test.go @@ -344,14 +344,14 @@ func TestCheckAndHeal(t *testing.T) { if test.bootstrapping { mrfc.On("CheckSentinelMonitor", sentinel, bootstrapMaster, bootstrapMasterPort).Once().Return(nil) } else { - mrfc.On("CheckSentinelMonitor", sentinel, master).Once().Return(nil) + mrfc.On("CheckSentinelMonitor", sentinel, master, "0").Once().Return(nil) } } else { if test.bootstrapping { mrfc.On("CheckSentinelMonitor", sentinel, bootstrapMaster, bootstrapMasterPort).Once().Return(errors.New("")) mrfh.On("NewSentinelMonitorWithPort", sentinel, bootstrapMaster, bootstrapMasterPort, rf).Once().Return(nil) } else { - mrfc.On("CheckSentinelMonitor", sentinel, master).Once().Return(errors.New("")) + mrfc.On("CheckSentinelMonitor", sentinel, master, "0").Once().Return(errors.New("")) mrfh.On("NewSentinelMonitor", sentinel, master, rf).Once().Return(nil) } } diff --git a/operator/redisfailover/service/check.go b/operator/redisfailover/service/check.go index f5d4a7f0f..ddce86ae8 100644 --- a/operator/redisfailover/service/check.go +++ b/operator/redisfailover/service/check.go @@ -3,6 +3,7 @@ package service import ( "errors" "fmt" + "strconv" "time" appsv1 "k8s.io/api/apps/v1" @@ -104,6 +105,7 @@ func (r *RedisFailoverChecker) CheckAllSlavesFromMaster(master string, rf *redis return err } + rport := getRedisPort(rf.Spec.Redis.Port) for _, rp := range rps.Items { if rp.Status.PodIP == master { err = r.setMasterLabelIfNecessary(rf.Namespace, rp) @@ -117,7 +119,7 @@ func (r *RedisFailoverChecker) CheckAllSlavesFromMaster(master string, rf *redis } } - slave, err := r.redisClient.GetSlaveOf(rp.Status.PodIP, password) + slave, err := r.redisClient.GetSlaveOf(rp.Status.PodIP, rport, password) if err != nil { r.logger.Errorf("Get slave of master failed, maybe this node is not ready, pod ip: %s", rp.Status.PodIP) return err @@ -181,8 +183,9 @@ func (r *RedisFailoverChecker) GetMasterIP(rf *redisfailoverv1.RedisFailover) (s } masters := []string{} + rport := getRedisPort(rf.Spec.Redis.Port) for _, rip := range rips { - master, err := r.redisClient.IsMaster(rip, password) + master, err := r.redisClient.IsMaster(rip, rport, password) if err != nil { r.logger.Errorf("Get redis info failed, maybe this node is not ready, pod ip: %s", rip) continue @@ -211,8 +214,9 @@ func (r *RedisFailoverChecker) GetNumberMasters(rf *redisfailoverv1.RedisFailove return nMasters, err } + rport := getRedisPort(rf.Spec.Redis.Port) for _, rip := range rips { - master, err := r.redisClient.IsMaster(rip, password) + master, err := r.redisClient.IsMaster(rip, rport, password) if err != nil { r.logger.Errorf("Get redis info failed, maybe this node is not ready, pod ip: %s", rip) continue @@ -288,9 +292,10 @@ func (r *RedisFailoverChecker) GetRedisesSlavesPods(rf *redisfailoverv1.RedisFai return redises, err } + rport := getRedisPort(rf.Spec.Redis.Port) for _, rp := range rps.Items { if rp.Status.Phase == corev1.PodRunning && rp.DeletionTimestamp == nil { // Only work with running - master, err := r.redisClient.IsMaster(rp.Status.PodIP, password) + master, err := r.redisClient.IsMaster(rp.Status.PodIP, rport, password) if err != nil { return []string{}, err } @@ -314,9 +319,10 @@ func (r *RedisFailoverChecker) GetRedisesMasterPod(rFailover *redisfailoverv1.Re return "", err } + rport := getRedisPort(rFailover.Spec.Redis.Port) for _, rp := range rps.Items { if rp.Status.Phase == corev1.PodRunning && rp.DeletionTimestamp == nil { // Only work with running - master, err := r.redisClient.IsMaster(rp.Status.PodIP, password) + master, err := r.redisClient.IsMaster(rp.Status.PodIP, rport, password) if err != nil { return "", err } @@ -370,5 +376,10 @@ func (r *RedisFailoverChecker) CheckRedisSlavesReady(ip string, rFailover *redis return false, err } - return r.redisClient.SlaveIsReady(ip, password) + port := getRedisPort(rFailover.Spec.Redis.Port) + return r.redisClient.SlaveIsReady(ip, port, password) +} + +func getRedisPort(p int32) string { + return strconv.Itoa(int(p)) } diff --git a/operator/redisfailover/service/check_test.go b/operator/redisfailover/service/check_test.go index 207cade29..297777d30 100644 --- a/operator/redisfailover/service/check_test.go +++ b/operator/redisfailover/service/check_test.go @@ -186,7 +186,7 @@ func TestCheckAllSlavesFromMasterGetSlaveOfError(t *testing.T) { ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil) mr := &mRedisService.Client{} - mr.On("GetSlaveOf", "", "").Once().Return("", errors.New("")) + mr.On("GetSlaveOf", "", "0", "").Once().Return("", errors.New("")) checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}) @@ -214,7 +214,7 @@ func TestCheckAllSlavesFromMasterDifferentMaster(t *testing.T) { ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil) mr := &mRedisService.Client{} - mr.On("GetSlaveOf", "0.0.0.0", "").Once().Return("1.1.1.1", nil) + mr.On("GetSlaveOf", "0.0.0.0", "0", "").Once().Return("1.1.1.1", nil) checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}) @@ -242,7 +242,7 @@ func TestCheckAllSlavesFromMaster(t *testing.T) { ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil) mr := &mRedisService.Client{} - mr.On("GetSlaveOf", "0.0.0.0", "").Once().Return("1.1.1.1", nil) + mr.On("GetSlaveOf", "0.0.0.0", "0", "").Once().Return("1.1.1.1", nil) checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}) @@ -468,7 +468,7 @@ func TestGetMasterIPIsMasterError(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) mr := &mRedisService.Client{} - mr.On("IsMaster", "0.0.0.0", "").Once().Return(false, errors.New("")) + mr.On("IsMaster", "0.0.0.0", "0", "").Once().Return(false, errors.New("")) checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}) @@ -501,8 +501,8 @@ func TestGetMasterIPMultipleMastersError(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) mr := &mRedisService.Client{} - mr.On("IsMaster", "0.0.0.0", "").Once().Return(true, nil) - mr.On("IsMaster", "1.1.1.1", "").Once().Return(true, nil) + mr.On("IsMaster", "0.0.0.0", "0", "").Once().Return(true, nil) + mr.On("IsMaster", "1.1.1.1", "0", "").Once().Return(true, nil) checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}) @@ -535,8 +535,8 @@ func TestGetMasterIP(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) mr := &mRedisService.Client{} - mr.On("IsMaster", "0.0.0.0", "").Once().Return(true, nil) - mr.On("IsMaster", "1.1.1.1", "").Once().Return(false, nil) + mr.On("IsMaster", "0.0.0.0", "0", "").Once().Return(true, nil) + mr.On("IsMaster", "1.1.1.1", "0", "").Once().Return(false, nil) checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}) @@ -579,7 +579,7 @@ func TestGetNumberMastersIsMasterError(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) mr := &mRedisService.Client{} - mr.On("IsMaster", "0.0.0.0", "").Once().Return(true, errors.New("")) + mr.On("IsMaster", "0.0.0.0", "0", "").Once().Return(true, errors.New("")) checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}) @@ -612,8 +612,8 @@ func TestGetNumberMasters(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) mr := &mRedisService.Client{} - mr.On("IsMaster", "0.0.0.0", "").Once().Return(true, nil) - mr.On("IsMaster", "1.1.1.1", "").Once().Return(false, nil) + mr.On("IsMaster", "0.0.0.0", "0", "").Once().Return(true, nil) + mr.On("IsMaster", "1.1.1.1", "0", "").Once().Return(false, nil) checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}) @@ -647,8 +647,8 @@ func TestGetNumberMastersTwo(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) mr := &mRedisService.Client{} - mr.On("IsMaster", "0.0.0.0", "").Once().Return(true, nil) - mr.On("IsMaster", "1.1.1.1", "").Once().Return(true, nil) + mr.On("IsMaster", "0.0.0.0", "0", "").Once().Return(true, nil) + mr.On("IsMaster", "1.1.1.1", "0", "").Once().Return(true, nil) checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}) @@ -752,8 +752,8 @@ func TestGetRedisPodsNames(t *testing.T) { ms := &mK8SService.Services{} ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) mr := &mRedisService.Client{} - mr.On("IsMaster", "0.0.0.0", "").Twice().Return(false, nil) - mr.On("IsMaster", "1.1.1.1", "").Once().Return(true, nil) + mr.On("IsMaster", "0.0.0.0", "0", "").Twice().Return(false, nil) + mr.On("IsMaster", "1.1.1.1", "0", "").Once().Return(true, nil) checker := rfservice.NewRedisFailoverChecker(ms, mr, log.DummyLogger{}) master, err := checker.GetRedisesMasterPod(rf) @@ -763,8 +763,8 @@ func TestGetRedisPodsNames(t *testing.T) { assert.Equal(master, "master") ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) - mr.On("IsMaster", "0.0.0.0", "").Twice().Return(false, nil) - mr.On("IsMaster", "1.1.1.1", "").Once().Return(true, nil) + mr.On("IsMaster", "0.0.0.0", "0", "").Twice().Return(false, nil) + mr.On("IsMaster", "1.1.1.1", "0", "").Once().Return(true, nil) namePods, err := checker.GetRedisesSlavesPods(rf) diff --git a/operator/redisfailover/service/generator.go b/operator/redisfailover/service/generator.go index 5690e7be8..24f4613ea 100644 --- a/operator/redisfailover/service/generator.go +++ b/operator/redisfailover/service/generator.go @@ -20,7 +20,8 @@ import ( const ( redisConfigurationVolumeName = "redis-config" // Template used to build the Redis configuration - redisConfigTemplate = `slaveof 127.0.0.1 6379 + redisConfigTemplate = `slaveof 127.0.0.1 {{.Spec.Redis.Port}} +port {{.Spec.Redis.Port}} tcp-keepalive 60 save 900 1 save 300 10 @@ -29,6 +30,12 @@ user pinger -@all +ping on >pingpass rename-command "{{.From}}" "{{.To}}" {{- end}} ` + + sentinelConfigTemplate = `sentinel monitor mymaster 127.0.0.1 {{.Spec.Redis.Port}} 2 +sentinel down-after-milliseconds mymaster 1000 +sentinel failover-timeout mymaster 3000 +sentinel parallel-syncs mymaster 2` + redisShutdownConfigurationVolumeName = "redis-shutdown-config" redisReadinessVolumeName = "redis-readiness-config" redisStorageVolumeName = "redis-data" @@ -107,10 +114,18 @@ func generateSentinelConfigMap(rf *redisfailoverv1.RedisFailover, labels map[str namespace := rf.Namespace labels = util.MergeLabels(labels, generateSelectorLabels(sentinelRoleName, rf.Name)) - sentinelConfigFileContent := `sentinel monitor mymaster 127.0.0.1 6379 2 -sentinel down-after-milliseconds mymaster 1000 -sentinel failover-timeout mymaster 3000 -sentinel parallel-syncs mymaster 2` + + tmpl, err := template.New("sentinel").Parse(sentinelConfigTemplate) + if err != nil { + panic(err) + } + + var tplOutput bytes.Buffer + if err := tmpl.Execute(&tplOutput, rf); err != nil { + panic(err) + } + + sentinelConfigFileContent := tplOutput.String() return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -160,15 +175,22 @@ func generateRedisConfigMap(rf *redisfailoverv1.RedisFailover, labels map[string func generateRedisShutdownConfigMap(rf *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) *corev1.ConfigMap { name := GetRedisShutdownConfigMapName(rf) + port := rf.Spec.Redis.Port namespace := rf.Namespace rfName := strings.Replace(strings.ToUpper(rf.Name), "-", "_", -1) labels = util.MergeLabels(labels, generateSelectorLabels(redisRoleName, rf.Name)) shutdownContent := fmt.Sprintf(`master=$(redis-cli -h ${RFS_%[1]v_SERVICE_HOST} -p ${RFS_%[1]v_SERVICE_PORT_SENTINEL} --csv SENTINEL get-master-addr-by-name mymaster | tr ',' ' ' | tr -d '\"' |cut -d' ' -f1) -redis-cli SAVE if [ "$master" = "$(hostname -i)" ]; then redis-cli -h ${RFS_%[1]v_SERVICE_HOST} -p ${RFS_%[1]v_SERVICE_PORT_SENTINEL} SENTINEL failover mymaster -fi`, rfName) + sleep 31 +fi +cmd="redis-cli -p %[2]v" +if [ ! -z "${REDIS_PASSWORD}" ]; then + cmd="${cmd} --no-auth-warning -a \"${REDIS_PASSWORD}\"" +fi +save_command="${cmd} save" +eval $save_command`, rfName, port) return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -185,49 +207,50 @@ fi`, rfName) func generateRedisReadinessConfigMap(rf *redisfailoverv1.RedisFailover, labels map[string]string, ownerRefs []metav1.OwnerReference) *corev1.ConfigMap { name := GetRedisReadinessName(rf) + port := rf.Spec.Redis.Port namespace := rf.Namespace labels = util.MergeLabels(labels, generateSelectorLabels(redisRoleName, rf.Name)) - readinessContent := `ROLE="role" - ROLE_MASTER="role:master" - ROLE_SLAVE="role:slave" - IN_SYNC="master_sync_in_progress:1" - NO_MASTER="master_host:127.0.0.1" - - cmd="redis-cli" - if [ ! -z "${REDIS_PASSWORD}" ]; then - cmd="${cmd} --no-auth-warning -a \"${REDIS_PASSWORD}\"" - fi - - cmd="${cmd} info replication" - - check_master(){ - exit 0 - } - - check_slave(){ - in_sync=$(echo "${cmd} | grep ${IN_SYNC} | tr -d \"\\r\" | tr -d \"\\n\"" | xargs -0 sh -c) - no_master=$(echo "${cmd} | grep ${NO_MASTER} | tr -d \"\\r\" | tr -d \"\\n\"" | xargs -0 sh -c) - - if [ -z "$in_sync" ] && [ -z "$no_master" ]; then - exit 0 - fi - - exit 1 - } - - role=$(echo "${cmd} | grep $ROLE | tr -d \"\\r\" | tr -d \"\\n\"" | xargs -0 sh -c) - case $role in - $ROLE_MASTER) - check_master - ;; - $ROLE_SLAVE) - check_slave - ;; - *) - echo "unespected" - exit 1 - esac` + readinessContent := fmt.Sprintf(`ROLE="role" +ROLE_MASTER="role:master" +ROLE_SLAVE="role:slave" +IN_SYNC="master_sync_in_progress:1" +NO_MASTER="master_host:127.0.0.1" + +cmd="redis-cli -p %[1]v" +if [ ! -z "${REDIS_PASSWORD}" ]; then + cmd="${cmd} --no-auth-warning -a \"${REDIS_PASSWORD}\"" +fi + +cmd="${cmd} info replication" + +check_master(){ + exit 0 +} + +check_slave(){ + in_sync=$(echo "${cmd} | grep ${IN_SYNC} | tr -d \"\\r\" | tr -d \"\\n\"" | xargs -0 sh -c) + no_master=$(echo "${cmd} | grep ${NO_MASTER} | tr -d \"\\r\" | tr -d \"\\n\"" | xargs -0 sh -c) + + if [ -z "$in_sync" ] && [ -z "$no_master" ]; then + exit 0 + fi + + exit 1 +} + +role=$(echo "${cmd} | grep $ROLE | tr -d \"\\r\" | tr -d \"\\n\"" | xargs -0 sh -c) +case $role in + $ROLE_MASTER) + check_master + ;; + $ROLE_SLAVE) + check_slave + ;; + *) + echo "unespected" + exit 1 +esac`, port) return &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -297,7 +320,7 @@ func generateRedisStatefulSet(rf *redisfailoverv1.RedisFailover, labels map[stri Ports: []corev1.ContainerPort{ { Name: "redis", - ContainerPort: 6379, + ContainerPort: rf.Spec.Redis.Port, Protocol: corev1.ProtocolTCP, }, }, @@ -322,7 +345,7 @@ func generateRedisStatefulSet(rf *redisfailoverv1.RedisFailover, labels map[stri Command: []string{ "sh", "-c", - "redis-cli -h $(hostname) ping --user pinger --pass pingpass --no-auth-warning", + fmt.Sprintf("redis-cli -h $(hostname) -p %[1]v ping --user pinger --pass pingpass --no-auth-warning", rf.Spec.Redis.Port), }, }, }, @@ -608,6 +631,13 @@ func createRedisExporterContainer(rf *redisfailoverv1.RedisFailover) corev1.Cont }) } + if rf.Spec.Redis.Port != 6379 { + container.Env = append(container.Env, corev1.EnvVar{ + Name: "REDIS_ADDR", + Value: fmt.Sprintf("redis://localhost:%[1]v", rf.Spec.Redis.Port), + }) + } + return container } diff --git a/operator/redisfailover/service/heal.go b/operator/redisfailover/service/heal.go index acda1da5b..934915d9d 100644 --- a/operator/redisfailover/service/heal.go +++ b/operator/redisfailover/service/heal.go @@ -66,7 +66,8 @@ func (r *RedisFailoverHealer) MakeMaster(ip string, rf *redisfailoverv1.RedisFai return err } - err = r.redisClient.MakeMaster(ip, password) + port := getRedisPort(rf.Spec.Redis.Port) + err = r.redisClient.MakeMaster(ip, port, password) if err != nil { return err } @@ -103,11 +104,13 @@ func (r *RedisFailoverHealer) SetOldestAsMaster(rf *redisfailoverv1.RedisFailove return err } + port := getRedisPort(rf.Spec.Redis.Port) newMasterIP := "" for _, pod := range ssp.Items { if newMasterIP == "" { - r.logger.Infof("New master is %s with ip %s", pod.Name, pod.Status.PodIP) - if err := r.redisClient.MakeMaster(pod.Status.PodIP, password); err != nil { + newMasterIP = pod.Status.PodIP + r.logger.Debugf("New master is %s with ip %s", pod.Name, newMasterIP) + if err := r.redisClient.MakeMaster(newMasterIP, port, password); err != nil { r.logger.Errorf("Make new master failed, master ip: %s, error: %v", pod.Status.PodIP, err) continue } @@ -119,8 +122,8 @@ func (r *RedisFailoverHealer) SetOldestAsMaster(rf *redisfailoverv1.RedisFailove newMasterIP = pod.Status.PodIP } else { - r.logger.Infof("Making pod %s slave of %s", pod.Name, newMasterIP) - if err := r.redisClient.MakeSlaveOf(pod.Status.PodIP, newMasterIP, password); err != nil { + r.logger.Debugf("Making pod %s slave of %s", pod.Name, newMasterIP) + if err := r.redisClient.MakeSlaveOfWithPort(pod.Status.PodIP, newMasterIP, port, password); err != nil { r.logger.Errorf("Make slave failed, slave pod ip: %s, master ip: %s, error: %v", pod.Status.PodIP, newMasterIP, err) } @@ -145,10 +148,11 @@ func (r *RedisFailoverHealer) SetMasterOnAll(masterIP string, rf *redisfailoverv return err } + port := getRedisPort(rf.Spec.Redis.Port) for _, pod := range ssp.Items { if pod.Status.PodIP == masterIP { - r.logger.Infof("Ensure pod %s is master", pod.Name) - if err := r.redisClient.MakeMaster(masterIP, password); err != nil { + r.logger.Debugf("Ensure pod %s is master", pod.Name) + if err := r.redisClient.MakeMaster(masterIP, port, password); err != nil { r.logger.Errorf("Make master failed, master ip: %s, error: %v", masterIP, err) return err } @@ -158,8 +162,8 @@ func (r *RedisFailoverHealer) SetMasterOnAll(masterIP string, rf *redisfailoverv return err } } else { - r.logger.Infof("Making pod %s slave of %s", pod.Name, masterIP) - if err := r.redisClient.MakeSlaveOf(pod.Status.PodIP, masterIP, password); err != nil { + r.logger.Debugf("Making pod %s slave of %s", pod.Name, masterIP) + if err := r.redisClient.MakeSlaveOfWithPort(pod.Status.PodIP, masterIP, port, password); err != nil { r.logger.Errorf("Make slave failed, slave ip: %s, master ip: %s, error: %v", pod.Status.PodIP, masterIP, err) return err } @@ -206,7 +210,8 @@ func (r *RedisFailoverHealer) NewSentinelMonitor(ip string, monitor string, rf * return err } - return r.redisClient.MonitorRedis(ip, monitor, quorum, password) + port := getRedisPort(rf.Spec.Redis.Port) + return r.redisClient.MonitorRedisWithPort(ip, monitor, port, quorum, password) } // NewSentinelMonitorWithPort changes the master that Sentinel has to monitor by the provided IP and Port @@ -243,7 +248,8 @@ func (r *RedisFailoverHealer) SetRedisCustomConfig(ip string, rf *redisfailoverv return err } - return r.redisClient.SetCustomRedisConfig(ip, rf.Spec.Redis.CustomConfig, password) + port := getRedisPort(rf.Spec.Redis.Port) + return r.redisClient.SetCustomRedisConfig(ip, port, rf.Spec.Redis.CustomConfig, password) } //DeletePod delete a failing pod so kubernetes relaunch it again diff --git a/operator/redisfailover/service/heal_test.go b/operator/redisfailover/service/heal_test.go index 5220a6621..f9feb8ea4 100644 --- a/operator/redisfailover/service/heal_test.go +++ b/operator/redisfailover/service/heal_test.go @@ -36,7 +36,7 @@ func TestSetOldestAsMasterNewMasterError(t *testing.T) { ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) mr := &mRedisService.Client{} - mr.On("MakeMaster", "0.0.0.0", "").Once().Return(errors.New("")) + mr.On("MakeMaster", "0.0.0.0", "0", "").Once().Return(errors.New("")) healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{}) @@ -63,7 +63,7 @@ func TestSetOldestAsMaster(t *testing.T) { ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil) mr := &mRedisService.Client{} - mr.On("MakeMaster", "0.0.0.0", "").Once().Return(nil) + mr.On("MakeMaster", "0.0.0.0", "0", "").Once().Return(nil) healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{}) @@ -95,8 +95,8 @@ func TestSetOldestAsMasterMultiplePodsMakeSlaveOfError(t *testing.T) { ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) mr := &mRedisService.Client{} - mr.On("MakeMaster", "0.0.0.0", "").Once().Return(nil) - mr.On("MakeSlaveOf", "1.1.1.1", "0.0.0.0", "").Once().Return(errors.New("")) + mr.On("MakeMaster", "0.0.0.0", "0", "").Once().Return(nil) + mr.On("MakeSlaveOfWithPort", "1.1.1.1", "0.0.0.0", "0", "").Once().Return(errors.New("")) healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{}) @@ -128,8 +128,8 @@ func TestSetOldestAsMasterMultiplePods(t *testing.T) { ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) mr := &mRedisService.Client{} - mr.On("MakeMaster", "0.0.0.0", "").Once().Return(nil) - mr.On("MakeSlaveOf", "1.1.1.1", "0.0.0.0", "").Once().Return(nil) + mr.On("MakeMaster", "0.0.0.0", "0", "").Once().Return(nil) + mr.On("MakeSlaveOfWithPort", "1.1.1.1", "0.0.0.0", "0", "").Once().Return(nil) healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{}) @@ -171,8 +171,8 @@ func TestSetOldestAsMasterOrdering(t *testing.T) { ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) mr := &mRedisService.Client{} - mr.On("MakeMaster", "1.1.1.1", "").Once().Return(nil) - mr.On("MakeSlaveOf", "0.0.0.0", "1.1.1.1", "").Once().Return(nil) + mr.On("MakeMaster", "1.1.1.1", "0", "").Once().Return(nil) + mr.On("MakeSlaveOfWithPort", "0.0.0.0", "1.1.1.1", "0", "").Once().Return(nil) healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{}) @@ -204,7 +204,7 @@ func TestSetMasterOnAllMakeMasterError(t *testing.T) { ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Once().Return(nil) mr := &mRedisService.Client{} - mr.On("MakeMaster", "0.0.0.0", "").Once().Return(errors.New("")) + mr.On("MakeMaster", "0.0.0.0", "0", "").Once().Return(errors.New("")) healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{}) @@ -236,8 +236,8 @@ func TestSetMasterOnAllMakeSlaveOfError(t *testing.T) { ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) mr := &mRedisService.Client{} - mr.On("MakeMaster", "0.0.0.0", "").Once().Return(nil) - mr.On("MakeSlaveOf", "1.1.1.1", "0.0.0.0", "").Once().Return(errors.New("")) + mr.On("MakeMaster", "0.0.0.0", "0", "").Once().Return(nil) + mr.On("MakeSlaveOfWithPort", "1.1.1.1", "0.0.0.0", "0", "").Once().Return(errors.New("")) healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{}) @@ -269,8 +269,8 @@ func TestSetMasterOnAll(t *testing.T) { ms.On("GetStatefulSetPods", namespace, rfservice.GetRedisName(rf)).Once().Return(pods, nil) ms.On("UpdatePodLabels", namespace, mock.AnythingOfType("string"), mock.Anything).Return(nil) mr := &mRedisService.Client{} - mr.On("MakeMaster", "0.0.0.0", "").Once().Return(nil) - mr.On("MakeSlaveOf", "1.1.1.1", "0.0.0.0", "").Once().Return(nil) + mr.On("MakeMaster", "0.0.0.0", "0", "").Once().Return(nil) + mr.On("MakeSlaveOfWithPort", "1.1.1.1", "0.0.0.0", "0", "").Once().Return(nil) healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{}) @@ -376,9 +376,9 @@ func TestNewSentinelMonitor(t *testing.T) { if test.errorOnMonitorRedis { errorExpected = true - mr.On("MonitorRedis", "0.0.0.0", "1.1.1.1", "2", "").Once().Return(errors.New("")) + mr.On("MonitorRedisWithPort", "0.0.0.0", "1.1.1.1", "0", "2", "").Once().Return(errors.New("")) } else { - mr.On("MonitorRedis", "0.0.0.0", "1.1.1.1", "2", "").Once().Return(nil) + mr.On("MonitorRedisWithPort", "0.0.0.0", "1.1.1.1", "0", "2", "").Once().Return(nil) } healer := rfservice.NewRedisFailoverHealer(ms, mr, log.DummyLogger{}) diff --git a/service/redis/client.go b/service/redis/client.go index 036bfebe3..1c3ccafe6 100644 --- a/service/redis/client.go +++ b/service/redis/client.go @@ -16,17 +16,17 @@ type Client interface { GetNumberSentinelsInMemory(ip string) (int32, error) GetNumberSentinelSlavesInMemory(ip string) (int32, error) ResetSentinel(ip string) error - GetSlaveOf(ip, password string) (string, error) - IsMaster(ip, password string) (bool, error) + GetSlaveOf(ip, port, password string) (string, error) + IsMaster(ip, port, password string) (bool, error) MonitorRedis(ip, monitor, quorum, password string) error MonitorRedisWithPort(ip, monitor, port, quorum, password string) error - MakeMaster(ip, password string) error + MakeMaster(ip, port, password string) error MakeSlaveOf(ip, masterIP, password string) error MakeSlaveOfWithPort(ip, masterIP, masterPort, password string) error GetSentinelMonitor(ip string) (string, string, error) SetCustomSentinelConfig(ip string, configs []string) error - SetCustomRedisConfig(ip string, configs []string, password string) error - SlaveIsReady(ip, password string) (bool, error) + SetCustomRedisConfig(ip string, port string, configs []string, password string) error + SlaveIsReady(ip, port, password string) (bool, error) } type client struct { @@ -142,9 +142,9 @@ func (c *client) ResetSentinel(ip string) error { } // GetSlaveOf returns the master of the given redis, or nil if it's master -func (c *client) GetSlaveOf(ip, password string) (string, error) { +func (c *client) GetSlaveOf(ip, port, password string) (string, error) { options := &rediscli.Options{ - Addr: net.JoinHostPort(ip, redisPort), + Addr: net.JoinHostPort(ip, port), Password: password, DB: 0, } @@ -161,9 +161,9 @@ func (c *client) GetSlaveOf(ip, password string) (string, error) { return match[1], nil } -func (c *client) IsMaster(ip, password string) (bool, error) { +func (c *client) IsMaster(ip, port, password string) (bool, error) { options := &rediscli.Options{ - Addr: net.JoinHostPort(ip, redisPort), + Addr: net.JoinHostPort(ip, port), Password: password, DB: 0, } @@ -215,9 +215,9 @@ func (c *client) MonitorRedisWithPort(ip, monitor, port, quorum, password string return nil } -func (c *client) MakeMaster(ip string, password string) error { +func (c *client) MakeMaster(ip string, port string, password string) error { options := &rediscli.Options{ - Addr: net.JoinHostPort(ip, redisPort), + Addr: net.JoinHostPort(ip, port), Password: password, DB: 0, } @@ -235,7 +235,7 @@ func (c *client) MakeSlaveOf(ip, masterIP, password string) error { func (c *client) MakeSlaveOfWithPort(ip, masterIP, masterPort, password string) error { options := &rediscli.Options{ - Addr: net.JoinHostPort(ip, redisPort), // this is IP and Port for the RedisFailover redis + Addr: net.JoinHostPort(ip, masterPort), // this is IP and Port for the RedisFailover redis Password: password, DB: 0, } @@ -290,9 +290,9 @@ func (c *client) SetCustomSentinelConfig(ip string, configs []string) error { return nil } -func (c *client) SetCustomRedisConfig(ip string, configs []string, password string) error { +func (c *client) SetCustomRedisConfig(ip string, port string, configs []string, password string) error { options := &rediscli.Options{ - Addr: net.JoinHostPort(ip, redisPort), + Addr: net.JoinHostPort(ip, port), Password: password, DB: 0, } @@ -333,9 +333,9 @@ func (c *client) getConfigParameters(config string) (parameter string, value str return s[0], strings.Join(s[1:], " "), nil } -func (c *client) SlaveIsReady(ip, password string) (bool, error) { +func (c *client) SlaveIsReady(ip, port, password string) (bool, error) { options := &rediscli.Options{ - Addr: net.JoinHostPort(ip, redisPort), + Addr: net.JoinHostPort(ip, port), Password: password, DB: 0, } diff --git a/test/integration/redisfailover/creation_test.go b/test/integration/redisfailover/creation_test.go index 8ba731ea0..421c3e220 100644 --- a/test/integration/redisfailover/creation_test.go +++ b/test/integration/redisfailover/creation_test.go @@ -215,7 +215,7 @@ func (c *clients) testRedisMaster(t *testing.T) { for _, pod := range redisPodList.Items { ip := pod.Status.PodIP - if ok, _ := c.redisClient.IsMaster(ip, testPass); ok { + if ok, _ := c.redisClient.IsMaster(ip, "6379", testPass); ok { masters = append(masters, ip) } } @@ -246,7 +246,7 @@ func (c *clients) testSentinelMonitoring(t *testing.T) { assert.Equal(masters[0], masterIP, "all master ip monitoring should equal") } - isMaster, err := c.redisClient.IsMaster(masters[0], testPass) + isMaster, err := c.redisClient.IsMaster(masters[0], "6379", testPass) assert.NoError(err) assert.True(isMaster, "Sentinel should monitor the Redis master") }