stable/redis-ha: Fix graceful Sentinel failover

charts/redis-ha/Chart.yaml

@@ -5,7 +5,7 @@ keywords:
 - redis
 - keyvalue
 - database
-version: 4.35.5
+version: 4.35.6
 appVersion: 8.2.2
 description: This Helm chart provides a highly available Redis implementation with a master/slave configuration and uses Sentinel sidecars for failover management
 icon: https://img.icons8.com/external-tal-revivo-shadow-tal-revivo/24/external-redis-an-in-memory-data-structure-project-implementing-a-distributed-logo-shadow-tal-revivo.png

charts/redis-ha/templates/_configs.tpl

@@ -295,6 +295,22 @@
         fi
     }
 
+    get_redis_role() {
+      is_master=$(
+        redis-cli \
+        {{- if .Values.auth }}
+          -a "${AUTH}" --no-auth-warning \
+        {{- end }}
+          -h localhost \
+        {{- if (int .Values.redis.port) }}
+          -p {{ .Values.redis.port }} \
+        {{- else }}
+          -p {{ .Values.redis.tlsPort }} ${TLS_CLIENT_OPTION} \
+        {{- end}}
+          info | grep -c 'role:master' || true
+      )
+    }
+
     redis_ro_update() {
         echo "Updating read-only redis config.."
         echo "  redis.conf set 'replica-priority 0'"
@@ -402,27 +418,18 @@
 {{- end }}
 
 {{- define "trigger-failover-if-master.sh" }}
+    #!/bin/sh
+    set -e
+
+    . $(dirname "$(realpath "$0")")/lib.sh
+
     {{- if or (eq (int .Values.redis.port) 0) (eq (int .Values.sentinel.port) 0) }}
     TLS_CLIENT_OPTION="--tls --cacert /tls-certs/{{ .Values.tls.caCertFile }}{{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{end}}"
     {{- end }}
-    get_redis_role() {
-      is_master=$(
-        redis-cli \
-        {{- if .Values.auth }}
-          -a "${AUTH}" --no-auth-warning \
-        {{- end }}
-          -h localhost \
-        {{- if (int .Values.redis.port) }}
-          -p {{ .Values.redis.port }} \
-        {{- else }}
-          -p {{ .Values.redis.tlsPort }} ${TLS_CLIENT_OPTION} \
-        {{- end}}
-          info | grep -c 'role:master' || true
-      )
-    }
+
     get_redis_role
     if [[ "$is_master" -eq 1 ]]; then
-      echo "This node is currently master, we trigger a failover."
+      echo "[$0] This node is currently master, we trigger a failover."
       {{- $masterGroupName := include "redis-ha.masterGroupName" . }}
       response=$(
         redis-cli \
@@ -438,19 +445,51 @@
           SENTINEL failover {{ $masterGroupName }}
       )
       if [[ "$response" != "OK" ]] ; then
+        echo "[$0] Sentinel failover failed"
         echo "$response"
         exit 1
       fi
-      timeout=30
-      while [[ "$is_master" -eq 1 && $timeout -gt 0 ]]; do
-        sleep 1
-        get_redis_role
-        timeout=$((timeout - 1))
-      done
-      echo "Failover successful"
+
+      # Give it some time to propagate the master change to the clients with pub/sub
+      sleep 2
+
+      # Disconnect all Sentinel clients before shutting down
+      redis-cli \
+      {{- if .Values.sentinel.auth }}
+          -a "${SENTINELAUTH}" --no-auth-warning \
+      {{- end }}
+          -h localhost \
+      {{- if (int .Values.sentinel.port) }}
+          -p {{ .Values.sentinel.port }} \
+      {{- else }}
+          -p {{ .Values.sentinel.tlsPort }} ${TLS_CLIENT_OPTION} \
+      {{- end}}
+          CLIENT KILL TYPE normal
+
+      echo "[$0] Sentinel failover successful"
     fi
 {{- end }}
 
+{{- define "redis-prestop.sh" }}
+    #!/bin/sh
+    set -e
+
+    . $(dirname "$(realpath "$0")")/lib.sh
+
+    {{- if or (eq (int .Values.redis.port) 0) (eq (int .Values.sentinel.port) 0) }}
+    TLS_CLIENT_OPTION="--tls --cacert /tls-certs/{{ .Values.tls.caCertFile }}{{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{end}}"
+    {{- end }}
+
+    timeout=30
+    get_redis_role
+    while [[ "$is_master" -eq 1 && $timeout -gt 0 ]]; do
+      echo "[$0] This Redis node is currently master, let's wait until this has changed (${timeout}s)."
+      sleep 1
+      get_redis_role
+      timeout=$((timeout - 1))
+    done
+{{- end }}
+
 {{- define "fix-split-brain.sh" }}
     {{- include "vars.sh" . }}
 

charts/redis-ha/templates/redis-ha-configmap.yaml

@@ -35,3 +35,7 @@ data:
 {{- include "config-haproxy_init.sh" . }}
   trigger-failover-if-master.sh: |
 {{- include "trigger-failover-if-master.sh" . }}
+  redis-prestop.sh: |
+{{- include "redis-prestop.sh" . }}
+  lib.sh: |
+{{- include "lib.sh" . }}

charts/redis-ha/templates/redis-ha-statefulset.yaml

@@ -31,7 +31,7 @@ spec:
   template:
     metadata:
       annotations:
-        checksum/init-config: {{ print (include "config-redis.conf" .) (include "config-sentinel.conf" .) (include "config-init.sh" .) (include "fix-split-brain.sh" .) (include "redis_liveness.sh" .) (include "redis_readiness.sh" .) (include "sentinel_liveness.sh" .) (include "trigger-failover-if-master.sh" .)| sha256sum }}
+        checksum/init-config: {{ print (include "config-redis.conf" .) (include "config-sentinel.conf" .) (include "config-init.sh" .) (include "fix-split-brain.sh" .) (include "redis_liveness.sh" .) (include "redis_readiness.sh" .) (include "sentinel_liveness.sh" .) (include "trigger-failover-if-master.sh" .) (include "redis-prestop.sh" .) | sha256sum }}
       {{- if .Values.redis.podAnnotations }}
 {{ toYaml .Values.redis.podAnnotations | indent 8 }}
       {{- end }}
@@ -475,6 +475,9 @@ spec:
         {{- end }}
         - mountPath: /health
           name: health
+        - name: config
+          mountPath: /readonly-config
+          readOnly: true
 {{- if .Values.sentinel.extraVolumeMounts }}
 {{- toYaml .Values.sentinel.extraVolumeMounts | nindent 8 }}
 {{- end }}

charts/redis-ha/values.yaml

@@ -494,7 +494,11 @@ redis:
   lifecycle:
     preStop:
       exec:
-        command: ["/bin/sh", "/readonly-config/trigger-failover-if-master.sh"]
+        command: [
+          "/bin/sh",
+          "-c",
+          "sh /readonly-config/redis-prestop.sh > /proc/1/fd/1 2> /proc/1/fd/2"
+        ]
 
   # -- Annotations for the redis statefulset
   annotations: {}
@@ -638,7 +642,14 @@ sentinel:
 
   # -- Container Lifecycle Hooks for sentinel container.
   # Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
-  lifecycle: {}
+  lifecycle:
+    preStop:
+      exec:
+        command: [
+          "/bin/sh",
+          "-c",
+          "sh /readonly-config/trigger-failover-if-master.sh > /proc/1/fd/1 2> /proc/1/fd/2"
+        ]
 
   # -- additional volumeMounts for Sentinel container
   extraVolumeMounts: []