diff --git a/charts/redis-ha/Chart.yaml b/charts/redis-ha/Chart.yaml index c646d30..397aa58 100644 --- a/charts/redis-ha/Chart.yaml +++ b/charts/redis-ha/Chart.yaml @@ -5,7 +5,7 @@ keywords: - redis - keyvalue - database -version: 4.35.5 +version: 4.35.6 appVersion: 8.2.2 description: This Helm chart provides a highly available Redis implementation with a master/slave configuration and uses Sentinel sidecars for failover management icon: https://img.icons8.com/external-tal-revivo-shadow-tal-revivo/24/external-redis-an-in-memory-data-structure-project-implementing-a-distributed-logo-shadow-tal-revivo.png diff --git a/charts/redis-ha/templates/_configs.tpl b/charts/redis-ha/templates/_configs.tpl index 83d30ad..bdc1133 100644 --- a/charts/redis-ha/templates/_configs.tpl +++ b/charts/redis-ha/templates/_configs.tpl @@ -295,6 +295,22 @@ fi } + get_redis_role() { + is_master=$( + redis-cli \ + {{- if .Values.auth }} + -a "${AUTH}" --no-auth-warning \ + {{- end }} + -h localhost \ + {{- if (int .Values.redis.port) }} + -p {{ .Values.redis.port }} \ + {{- else }} + -p {{ .Values.redis.tlsPort }} ${TLS_CLIENT_OPTION} \ + {{- end}} + info | grep -c 'role:master' || true + ) + } + redis_ro_update() { echo "Updating read-only redis config.." echo " redis.conf set 'replica-priority 0'" @@ -402,27 +418,18 @@ {{- end }} {{- define "trigger-failover-if-master.sh" }} + #!/bin/sh + set -e + + . $(dirname "$(realpath "$0")")/lib.sh + {{- if or (eq (int .Values.redis.port) 0) (eq (int .Values.sentinel.port) 0) }} TLS_CLIENT_OPTION="--tls --cacert /tls-certs/{{ .Values.tls.caCertFile }}{{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{end}}" {{- end }} - get_redis_role() { - is_master=$( - redis-cli \ - {{- if .Values.auth }} - -a "${AUTH}" --no-auth-warning \ - {{- end }} - -h localhost \ - {{- if (int .Values.redis.port) }} - -p {{ .Values.redis.port }} \ - {{- else }} - -p {{ .Values.redis.tlsPort }} ${TLS_CLIENT_OPTION} \ - {{- end}} - info | grep -c 'role:master' || true - ) - } + get_redis_role if [[ "$is_master" -eq 1 ]]; then - echo "This node is currently master, we trigger a failover." + echo "[$0] This node is currently master, we trigger a failover." {{- $masterGroupName := include "redis-ha.masterGroupName" . }} response=$( redis-cli \ @@ -438,19 +445,51 @@ SENTINEL failover {{ $masterGroupName }} ) if [[ "$response" != "OK" ]] ; then + echo "[$0] Sentinel failover failed" echo "$response" exit 1 fi - timeout=30 - while [[ "$is_master" -eq 1 && $timeout -gt 0 ]]; do - sleep 1 - get_redis_role - timeout=$((timeout - 1)) - done - echo "Failover successful" + + # Give it some time to propagate the master change to the clients with pub/sub + sleep 2 + + # Disconnect all Sentinel clients before shutting down + redis-cli \ + {{- if .Values.sentinel.auth }} + -a "${SENTINELAUTH}" --no-auth-warning \ + {{- end }} + -h localhost \ + {{- if (int .Values.sentinel.port) }} + -p {{ .Values.sentinel.port }} \ + {{- else }} + -p {{ .Values.sentinel.tlsPort }} ${TLS_CLIENT_OPTION} \ + {{- end}} + CLIENT KILL TYPE normal + + echo "[$0] Sentinel failover successful" fi {{- end }} +{{- define "redis-prestop.sh" }} + #!/bin/sh + set -e + + . $(dirname "$(realpath "$0")")/lib.sh + + {{- if or (eq (int .Values.redis.port) 0) (eq (int .Values.sentinel.port) 0) }} + TLS_CLIENT_OPTION="--tls --cacert /tls-certs/{{ .Values.tls.caCertFile }}{{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{end}}" + {{- end }} + + timeout=30 + get_redis_role + while [[ "$is_master" -eq 1 && $timeout -gt 0 ]]; do + echo "[$0] This Redis node is currently master, let's wait until this has changed (${timeout}s)." + sleep 1 + get_redis_role + timeout=$((timeout - 1)) + done +{{- end }} + {{- define "fix-split-brain.sh" }} {{- include "vars.sh" . }} diff --git a/charts/redis-ha/templates/redis-ha-configmap.yaml b/charts/redis-ha/templates/redis-ha-configmap.yaml index 48f2b03..40b54c4 100644 --- a/charts/redis-ha/templates/redis-ha-configmap.yaml +++ b/charts/redis-ha/templates/redis-ha-configmap.yaml @@ -35,3 +35,7 @@ data: {{- include "config-haproxy_init.sh" . }} trigger-failover-if-master.sh: | {{- include "trigger-failover-if-master.sh" . }} + redis-prestop.sh: | +{{- include "redis-prestop.sh" . }} + lib.sh: | +{{- include "lib.sh" . }} diff --git a/charts/redis-ha/templates/redis-ha-statefulset.yaml b/charts/redis-ha/templates/redis-ha-statefulset.yaml index eeb54c2..78c358d 100644 --- a/charts/redis-ha/templates/redis-ha-statefulset.yaml +++ b/charts/redis-ha/templates/redis-ha-statefulset.yaml @@ -31,7 +31,7 @@ spec: template: metadata: annotations: - checksum/init-config: {{ print (include "config-redis.conf" .) (include "config-sentinel.conf" .) (include "config-init.sh" .) (include "fix-split-brain.sh" .) (include "redis_liveness.sh" .) (include "redis_readiness.sh" .) (include "sentinel_liveness.sh" .) (include "trigger-failover-if-master.sh" .)| sha256sum }} + checksum/init-config: {{ print (include "config-redis.conf" .) (include "config-sentinel.conf" .) (include "config-init.sh" .) (include "fix-split-brain.sh" .) (include "redis_liveness.sh" .) (include "redis_readiness.sh" .) (include "sentinel_liveness.sh" .) (include "trigger-failover-if-master.sh" .) (include "redis-prestop.sh" .) | sha256sum }} {{- if .Values.redis.podAnnotations }} {{ toYaml .Values.redis.podAnnotations | indent 8 }} {{- end }} @@ -475,6 +475,9 @@ spec: {{- end }} - mountPath: /health name: health + - name: config + mountPath: /readonly-config + readOnly: true {{- if .Values.sentinel.extraVolumeMounts }} {{- toYaml .Values.sentinel.extraVolumeMounts | nindent 8 }} {{- end }} diff --git a/charts/redis-ha/values.yaml b/charts/redis-ha/values.yaml index 06180a7..abfe6fe 100644 --- a/charts/redis-ha/values.yaml +++ b/charts/redis-ha/values.yaml @@ -494,7 +494,11 @@ redis: lifecycle: preStop: exec: - command: ["/bin/sh", "/readonly-config/trigger-failover-if-master.sh"] + command: [ + "/bin/sh", + "-c", + "sh /readonly-config/redis-prestop.sh > /proc/1/fd/1 2> /proc/1/fd/2" + ] # -- Annotations for the redis statefulset annotations: {} @@ -638,7 +642,14 @@ sentinel: # -- Container Lifecycle Hooks for sentinel container. # Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/ - lifecycle: {} + lifecycle: + preStop: + exec: + command: [ + "/bin/sh", + "-c", + "sh /readonly-config/trigger-failover-if-master.sh > /proc/1/fd/1 2> /proc/1/fd/2" + ] # -- additional volumeMounts for Sentinel container extraVolumeMounts: []