From 012d80db544a0cfbf26a811f3aea854e1abc44f5 Mon Sep 17 00:00:00 2001 From: Alexander Sinuskin Date: Wed, 27 Nov 2024 16:47:38 +0100 Subject: [PATCH 1/7] Add option to have /healthz probes to be tied to database(s) connectivity --- main.go | 42 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/main.go b/main.go index 30319b12..40cc37db 100644 --- a/main.go +++ b/main.go @@ -22,10 +22,11 @@ func init() { func main() { var ( - showVersion = flag.Bool("version", false, "Print version information.") - listenAddress = flag.String("web.listen-address", ":9237", "Address to listen on for web interface and telemetry.") - metricsPath = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.") - configFile = flag.String("config.file", os.Getenv("CONFIG"), "SQL Exporter configuration file name.") + showVersion = flag.Bool("version", false, "Print version information.") + listenAddress = flag.String("web.listen-address", ":9237", "Address to listen on for web interface and telemetry.") + metricsPath = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.") + configFile = flag.String("config.file", os.Getenv("CONFIG"), "SQL Exporter configuration file name.") + dbConnectivityAsHealthCheck = flag.Bool("db.connectivity-as-healthz", false, "Use database connectivity check as healthz probe") ) flag.Parse() @@ -66,7 +67,38 @@ func main() { // setup and start webserver http.Handle(*metricsPath, promhttp.Handler()) - http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { http.Error(w, "OK", http.StatusOK) }) + + if *dbConnectivityAsHealthCheck { + http.HandleFunc("/healthz", + func(w http.ResponseWriter, r *http.Request) { + for _, job := range exporter.jobs { + if job == nil { + continue + } + for _, connection := range job.conns { + if connection == nil { + continue + } + if connection.conn == nil { + continue + } + + if err := connection.conn.Ping(); err != nil { + // if any of the connections fails to be established, fail the /healthz request + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + } + } + + // otherwise return OK + http.Error(w, "OK", http.StatusOK) + + }) + } else { + http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { http.Error(w, "OK", http.StatusOK) }) + } + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(` SQL Exporter From 7e848fb09a9c0949263393b4298de6e5f72fa987 Mon Sep 17 00:00:00 2001 From: Alexander Sinuskin Date: Mon, 2 Dec 2024 14:58:49 +0100 Subject: [PATCH 2/7] Just a NL --- .github/workflows/go.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 9eb6c8e4..441252ab 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -26,3 +26,4 @@ jobs: version: "2023.1" # go is already installed by actions/setup-go with the correct version from go.mod install-go: false + From 7c01c9d0ae9b89dabeb91cc85f4cb817ed0e2bcf Mon Sep 17 00:00:00 2001 From: Alexander Sinuskin Date: Tue, 3 Dec 2024 15:51:22 +0100 Subject: [PATCH 3/7] Debug-log found connections --- main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.go b/main.go index 40cc37db..a084060f 100644 --- a/main.go +++ b/main.go @@ -83,6 +83,8 @@ func main() { continue } + level.Debug(logger).Log("msg", "Non-null connection found", "connection", connection.conn) + if err := connection.conn.Ping(); err != nil { // if any of the connections fails to be established, fail the /healthz request http.Error(w, err.Error(), http.StatusInternalServerError) From 34bfd7afb4bae906caca5f5f4cb04347ae0946db Mon Sep 17 00:00:00 2001 From: Alexander Sinuskin Date: Tue, 3 Dec 2024 17:26:05 +0100 Subject: [PATCH 4/7] better debug logging --- main.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/main.go b/main.go index a084060f..0f2e2934 100644 --- a/main.go +++ b/main.go @@ -71,14 +71,25 @@ func main() { if *dbConnectivityAsHealthCheck { http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + level.Debug(logger).Log("msg", fmt.Sprintf("Healthz request received. Starting checking connections over %d jobs", len(exporter.jobs))) + for _, job := range exporter.jobs { + if job == nil { continue } + + level.Debug(logger).Log("msg", "Non-null job found", "job", job) + for _, connection := range job.conns { + level.Debug(logger).Log("msg", "Next job connection config", "connection", connection) + if connection == nil { continue } + + level.Debug(logger).Log("msg", "DB connection", "conn", connection.conn) + if connection.conn == nil { continue } From 144521f55620bf3e5da4f05e7cbc6954e44e2f43 Mon Sep 17 00:00:00 2001 From: Alexander Sinuskin Date: Tue, 3 Dec 2024 20:02:01 +0100 Subject: [PATCH 5/7] Fix the /healthz probe handler logic --- main.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/main.go b/main.go index 0f2e2934..9ad11ce7 100644 --- a/main.go +++ b/main.go @@ -71,36 +71,34 @@ func main() { if *dbConnectivityAsHealthCheck { http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { - level.Debug(logger).Log("msg", fmt.Sprintf("Healthz request received. Starting checking connections over %d jobs", len(exporter.jobs))) - for _, job := range exporter.jobs { if job == nil { continue } - level.Debug(logger).Log("msg", "Non-null job found", "job", job) - for _, connection := range job.conns { - level.Debug(logger).Log("msg", "Next job connection config", "connection", connection) if connection == nil { continue } - level.Debug(logger).Log("msg", "DB connection", "conn", connection.conn) - - if connection.conn == nil { + if connection.conn != nil { + if err := connection.conn.Ping(); err != nil { + // if any of the connections fails to be established/verified, fail the /healthz request + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + // otherwise we've successfully verified the connection, continue to the next one continue } - level.Debug(logger).Log("msg", "Non-null connection found", "connection", connection.conn) - - if err := connection.conn.Ping(); err != nil { + if err := connection.connect(job); err != nil { // if any of the connections fails to be established, fail the /healthz request http.Error(w, err.Error(), http.StatusInternalServerError) return } + } } From dc297c77699c53d07e554843a3e9ad552c156d77 Mon Sep 17 00:00:00 2001 From: Alexander Sinuskin Date: Wed, 4 Dec 2024 13:48:37 +0100 Subject: [PATCH 6/7] Revert the NL addition --- .github/workflows/go.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 441252ab..9eb6c8e4 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -26,4 +26,3 @@ jobs: version: "2023.1" # go is already installed by actions/setup-go with the correct version from go.mod install-go: false - From e4630e3a09c06703407703a82c849c694679f70c Mon Sep 17 00:00:00 2001 From: Firstname Lastname Date: Mon, 8 Sep 2025 17:25:21 +0200 Subject: [PATCH 7/7] Make the liveness call fail only when all configured jobs' connections failed to be established --- main.go | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/main.go b/main.go index 9ad11ce7..8a6bbdc7 100644 --- a/main.go +++ b/main.go @@ -85,26 +85,22 @@ func main() { if connection.conn != nil { if err := connection.conn.Ping(); err != nil { - // if any of the connections fails to be established/verified, fail the /healthz request - http.Error(w, err.Error(), http.StatusInternalServerError) - return + continue } - // otherwise we've successfully verified the connection, continue to the next one - continue + // if we meet at least one successful connection we consider the app healthy + http.Error(w, "OK", http.StatusOK) + return } - if err := connection.connect(job); err != nil { - // if any of the connections fails to be established, fail the /healthz request - http.Error(w, err.Error(), http.StatusInternalServerError) + if err := connection.connect(job); err == nil { + // if we meet at least one successful connection we consider the app healthy + http.Error(w, "OK", http.StatusOK) return } - } } - - // otherwise return OK - http.Error(w, "OK", http.StatusOK) - + // no successful connections found, fail the /healthz request + http.Error(w, err.Error(), http.StatusInternalServerError) }) } else { http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { http.Error(w, "OK", http.StatusOK) })