From d3867cc912d0a007cdf23a66476151e81fe135ec Mon Sep 17 00:00:00 2001 From: shaibuuneks Date: Wed, 17 Sep 2025 13:39:58 +0000 Subject: [PATCH 1/8] Prometheus + Grafana monitoring --- infra/Dockerfile | 4 + infra/grafana/Dockerfile | 5 ++ .../provisioning/dashboards/dashboards.yml | 11 +++ .../dashboards/nodejs-dashboard.json | 77 +++++++++++++++++++ .../provisioning/datasources/prometheus.yml | 8 ++ infra/prometheus/prometheus.yml | 11 +++ package-lock.json | 40 +++++++++- package.json | 3 +- render.yaml | 22 ++++++ src/app.js | 38 +++++++-- 10 files changed, 209 insertions(+), 10 deletions(-) create mode 100644 infra/Dockerfile create mode 100644 infra/grafana/Dockerfile create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yml create mode 100644 infra/grafana/provisioning/dashboards/nodejs-dashboard.json create mode 100644 infra/grafana/provisioning/datasources/prometheus.yml create mode 100644 infra/prometheus/prometheus.yml diff --git a/infra/Dockerfile b/infra/Dockerfile new file mode 100644 index 0000000..fa15349 --- /dev/null +++ b/infra/Dockerfile @@ -0,0 +1,4 @@ +FROM prom/prometheus:v2.53.0 + +# Copy custom config +COPY prometheus.yml /etc/prometheus/prometheus.yml diff --git a/infra/grafana/Dockerfile b/infra/grafana/Dockerfile new file mode 100644 index 0000000..d387315 --- /dev/null +++ b/infra/grafana/Dockerfile @@ -0,0 +1,5 @@ +FROM grafana/grafana:11.1.4 +FROM grafana/grafana:11.1.4 + +# Copy provisioning configs +COPY provisioning /etc/grafana/provisioning diff --git a/infra/grafana/provisioning/dashboards/dashboards.yml b/infra/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..636f645 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: "default" + orgId: 1 + folder: "" + type: file + disableDeletion: false + updateIntervalSeconds: 30 + options: + path: /etc/grafana/provisioning/dashboards diff --git a/infra/grafana/provisioning/dashboards/nodejs-dashboard.json b/infra/grafana/provisioning/dashboards/nodejs-dashboard.json new file mode 100644 index 0000000..8102539 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/nodejs-dashboard.json @@ -0,0 +1,77 @@ +{ + "id": null, + "title": "Node.js / Express Observability", + "tags": ["node", "express", "prometheus"], + "timezone": "browser", + "schemaVersion": 30, + "version": 1, + "panels": [ + { + "type": "graph", + "title": "HTTP Requests per Second", + "targets": [ + { + "expr": "rate(http_requests_total[1m])", + "legendFormat": "{{method}} {{route}} {{status}}" + } + ] + }, + { + "type": "graph", + "title": "Error Rate per Route", + "targets": [ + { + "expr": "rate(http_requests_total{status=~\"5..\"}[1m])", + "legendFormat": "{{method}} {{route}}" + } + ] + }, + { + "type": "graph", + "title": "CPU Usage %", + "targets": [ + { + "expr": "rate(process_cpu_user_seconds_total[1m]) * 100", + "legendFormat": "User CPU" + }, + { + "expr": "rate(process_cpu_system_seconds_total[1m]) * 100", + "legendFormat": "System CPU" + } + ], + "yaxes": [ + { "format": "percent", "min": 0, "max": 100 }, + { "format": "short" } + ] + }, + { + "type": "graph", + "title": "95th Percentile Request Latency", + "targets": [ + { + "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))", + "legendFormat": "{{method}} {{route}}" + } + ], + "yaxes": [ + { "format": "s", "min": 0 }, + { "format": "short" } + ] + }, + { + "type": "graph", + "title": "Memory Usage", + "targets": [ + { + "expr": "process_resident_memory_bytes", + "legendFormat": "Resident Memory" + } + ], + "yaxes": [ + { "format": "bytes", "min": 0 }, + { "format": "short" } + ] + } + ] + } + \ No newline at end of file diff --git a/infra/grafana/provisioning/datasources/prometheus.yml b/infra/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..86fd346 --- /dev/null +++ b/infra/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..045074e --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,11 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: "myapp" + metrics_path: /metrics + static_configs: + - targets: + # Replace with your staging/prod hostnames + ports + - "mydev-staging.onrender.com" + - "mydev-prod.onrender.com" diff --git a/package-lock.json b/package-lock.json index 7464de7..a376d8c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,8 @@ "dependencies": { "@sentry/node": "^7.120.4", "@sentry/tracing": "^7.120.4", - "express": "^4.21.2" + "express": "^4.21.2", + "prom-client": "^15.1.3" }, "devDependencies": { "eslint": "^8.57.1", @@ -1131,6 +1132,15 @@ "node": ">= 8" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@paralleldrive/cuid2": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/@paralleldrive/cuid2/-/cuid2-2.2.2.tgz", @@ -1677,6 +1687,12 @@ "baseline-browser-mapping": "dist/cli.js" } }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", + "license": "MIT" + }, "node_modules/body-parser": { "version": "1.20.3", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", @@ -5255,6 +5271,19 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/prom-client": { + "version": "15.1.3", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", + "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, "node_modules/prompts": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", @@ -6076,6 +6105,15 @@ "url": "https://opencollective.com/synckit" } }, + "node_modules/tdigest": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "license": "MIT", + "dependencies": { + "bintrees": "1.0.2" + } + }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", diff --git a/package.json b/package.json index 627923a..a7f2051 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,8 @@ "dependencies": { "@sentry/node": "^7.120.4", "@sentry/tracing": "^7.120.4", - "express": "^4.21.2" + "express": "^4.21.2", + "prom-client": "^15.1.3" }, "devDependencies": { "eslint": "^8.57.1", diff --git a/render.yaml b/render.yaml index 7db226d..b99a761 100644 --- a/render.yaml +++ b/render.yaml @@ -30,3 +30,25 @@ services: value: 10000 - key: SENTRY_DSN sync: false + + - type: web + name: prometheus + env: docker + plan: free + rootDir: ./infra/prometheus + dockerfilePath: ./infra/prometheus/Dockerfile + autoDeploy: false + envVars: + - key: PORT + value: 9090 + + - type: web + name: grafana + env: docker + plan: free + rootDir: ./infra/grafana + dockerfilePath: ./infra/grafana/Dockerfile + autoDeploy: false + envVars: + - key: PORT + value: 3000 diff --git a/src/app.js b/src/app.js index 11f2d61..8adbc12 100644 --- a/src/app.js +++ b/src/app.js @@ -1,19 +1,42 @@ const express = require("express"); const Sentry = require("@sentry/node"); const Tracing = require("@sentry/tracing"); +const client = require("prom-client"); const app = express(); -// Initialize Sentry (v7 API) +// --- Prometheus Setup --- +const collectDefaultMetrics = client.collectDefaultMetrics; +collectDefaultMetrics(); // Collect Node.js process metrics + +// Custom counter +const httpRequestCounter = new client.Counter({ + name: "http_requests_total", + help: "Total number of HTTP requests", + labelNames: ["method", "route", "status"], +}); + +// Middleware to count requests +app.use((req, res, next) => { + res.on("finish", () => { + httpRequestCounter.labels(req.method, req.path, res.statusCode).inc(); + }); + next(); +}); + +// Expose /metrics endpoint for Prometheus +app.get("/metrics", async (req, res) => { + res.set("Content-Type", client.register.contentType); + res.end(await client.register.metrics()); +}); + +// --- Sentry Setup (v7) --- Sentry.init({ dsn: process.env.SENTRY_DSN || "", - integrations: [ - new Tracing.Integrations.Express({ app }), // works fine with v7 - ], - tracesSampleRate: 1.0, // lower this in prod (e.g. 0.1) + integrations: [new Tracing.Integrations.Express({ app })], + tracesSampleRate: 1.0, }); -// Request handler (before routes) app.use(Sentry.Handlers.requestHandler()); app.use(Sentry.Handlers.tracingHandler()); @@ -22,13 +45,12 @@ app.get("/", (req, res) => { res.send("You are safe in Wizfi's Pipeline!"); }); -// Debug route to test Sentry (ESLint safe) +// Debug route app.get("/debug-sentry", (req, res) => { res.status(500).send("Triggering Sentry debug error..."); throw new Error("Debug Sentry error!"); }); -// Error handler (after routes) app.use(Sentry.Handlers.errorHandler()); module.exports = app; From b7c45ee8b51392c8ad93aeb805ef07832d929a42 Mon Sep 17 00:00:00 2001 From: shaibuuneks Date: Wed, 17 Sep 2025 13:50:16 +0000 Subject: [PATCH 2/8] Prometheus + Grafana monitoring --- render.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/render.yaml b/render.yaml index b99a761..69acf35 100644 --- a/render.yaml +++ b/render.yaml @@ -36,7 +36,7 @@ services: env: docker plan: free rootDir: ./infra/prometheus - dockerfilePath: ./infra/prometheus/Dockerfile + dockerfilePath: Dockerfile autoDeploy: false envVars: - key: PORT @@ -47,7 +47,7 @@ services: env: docker plan: free rootDir: ./infra/grafana - dockerfilePath: ./infra/grafana/Dockerfile + dockerfilePath: Dockerfile autoDeploy: false envVars: - key: PORT From 7ad66495e38f0f6520e8580ec0add624cdab1bd9 Mon Sep 17 00:00:00 2001 From: shaibuuneks Date: Wed, 17 Sep 2025 14:08:53 +0000 Subject: [PATCH 3/8] Prometheus + Grafana monitoring --- infra/{ => prometheus}/Dockerfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename infra/{ => prometheus}/Dockerfile (100%) diff --git a/infra/Dockerfile b/infra/prometheus/Dockerfile similarity index 100% rename from infra/Dockerfile rename to infra/prometheus/Dockerfile From 9b5cda79431376d5223d570376fa17a9be576062 Mon Sep 17 00:00:00 2001 From: shaibuuneks Date: Wed, 17 Sep 2025 14:41:10 +0000 Subject: [PATCH 4/8] prometheus fix --- render.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/render.yaml b/render.yaml index 69acf35..645089d 100644 --- a/render.yaml +++ b/render.yaml @@ -36,7 +36,7 @@ services: env: docker plan: free rootDir: ./infra/prometheus - dockerfilePath: Dockerfile + dockerfilePath: ./infra/prometheus/Dockerfile autoDeploy: false envVars: - key: PORT From a83a0ecf885bc15e948fcd5b30f66a5faa772437 Mon Sep 17 00:00:00 2001 From: shaibuuneks Date: Fri, 19 Sep 2025 07:17:50 +0000 Subject: [PATCH 5/8] prometheus deploy fix --- render.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/render.yaml b/render.yaml index 645089d..d90b0ab 100644 --- a/render.yaml +++ b/render.yaml @@ -36,7 +36,7 @@ services: env: docker plan: free rootDir: ./infra/prometheus - dockerfilePath: ./infra/prometheus/Dockerfile + dockerfilePath: Dockerfile autoDeploy: false envVars: - key: PORT From b81a2d594d28d8f64a8640268b614c094e0645b8 Mon Sep 17 00:00:00 2001 From: shaibuuneks Date: Fri, 19 Sep 2025 09:56:00 +0000 Subject: [PATCH 6/8] prometheus scraping --- .../dashboards/nodejs-dashboard.json | 146 +++++++++--------- infra/prometheus/prometheus.yml | 3 +- 2 files changed, 72 insertions(+), 77 deletions(-) diff --git a/infra/grafana/provisioning/dashboards/nodejs-dashboard.json b/infra/grafana/provisioning/dashboards/nodejs-dashboard.json index 8102539..2bf7d98 100644 --- a/infra/grafana/provisioning/dashboards/nodejs-dashboard.json +++ b/infra/grafana/provisioning/dashboards/nodejs-dashboard.json @@ -1,77 +1,71 @@ { - "id": null, - "title": "Node.js / Express Observability", - "tags": ["node", "express", "prometheus"], - "timezone": "browser", - "schemaVersion": 30, - "version": 1, - "panels": [ - { - "type": "graph", - "title": "HTTP Requests per Second", - "targets": [ - { - "expr": "rate(http_requests_total[1m])", - "legendFormat": "{{method}} {{route}} {{status}}" - } - ] - }, - { - "type": "graph", - "title": "Error Rate per Route", - "targets": [ - { - "expr": "rate(http_requests_total{status=~\"5..\"}[1m])", - "legendFormat": "{{method}} {{route}}" - } - ] - }, - { - "type": "graph", - "title": "CPU Usage %", - "targets": [ - { - "expr": "rate(process_cpu_user_seconds_total[1m]) * 100", - "legendFormat": "User CPU" - }, - { - "expr": "rate(process_cpu_system_seconds_total[1m]) * 100", - "legendFormat": "System CPU" - } - ], - "yaxes": [ - { "format": "percent", "min": 0, "max": 100 }, - { "format": "short" } - ] - }, - { - "type": "graph", - "title": "95th Percentile Request Latency", - "targets": [ - { - "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))", - "legendFormat": "{{method}} {{route}}" - } - ], - "yaxes": [ - { "format": "s", "min": 0 }, - { "format": "short" } - ] - }, - { - "type": "graph", - "title": "Memory Usage", - "targets": [ - { - "expr": "process_resident_memory_bytes", - "legendFormat": "Resident Memory" - } - ], - "yaxes": [ - { "format": "bytes", "min": 0 }, - { "format": "short" } - ] - } - ] - } - \ No newline at end of file + "id": null, + "title": "Node.js App Observability", + "tags": ["nodejs", "prometheus"], + "timezone": "browser", + "schemaVersion": 36, + "version": 1, + "refresh": "10s", + "panels": [ + { + "type": "graph", + "title": "CPU Usage (%)", + "targets": [ + { + "expr": "process_cpu_user_seconds_total", + "legendFormat": "CPU User Time", + "refId": "A" + } + ], + "id": 1 + }, + { + "type": "graph", + "title": "Memory Usage (MB)", + "targets": [ + { + "expr": "process_resident_memory_bytes / 1024 / 1024", + "legendFormat": "Resident Memory", + "refId": "A" + } + ], + "id": 2 + }, + { + "type": "graph", + "title": "HTTP Requests per Second", + "targets": [ + { + "expr": "rate(http_requests_total[1m])", + "legendFormat": "{{method}} {{route}}", + "refId": "A" + } + ], + "id": 3 + }, + { + "type": "graph", + "title": "Error Rate (5xx per Second)", + "targets": [ + { + "expr": "rate(http_requests_total{status=~\"5..\"}[5m])", + "legendFormat": "Errors (5xx)", + "refId": "A" + } + ], + "id": 4 + }, + { + "type": "graph", + "title": "95th Percentile Latency", + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p95 latency", + "refId": "A" + } + ], + "id": 5 + } + ] +} diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml index 045074e..3a08a22 100644 --- a/infra/prometheus/prometheus.yml +++ b/infra/prometheus/prometheus.yml @@ -1,11 +1,12 @@ global: scrape_interval: 15s + evaluation_interval: 15s scrape_configs: - job_name: "myapp" metrics_path: /metrics + scheme: https # Render apps use HTTPS static_configs: - targets: - # Replace with your staging/prod hostnames + ports - "mydev-staging.onrender.com" - "mydev-prod.onrender.com" From 890a667998d03a897c7a538861fc21d5575a6abb Mon Sep 17 00:00:00 2001 From: shaibuuneks Date: Fri, 19 Sep 2025 10:32:55 +0000 Subject: [PATCH 7/8] prometheus scraping --- infra/prometheus/prometheus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml index 3a08a22..96da60c 100644 --- a/infra/prometheus/prometheus.yml +++ b/infra/prometheus/prometheus.yml @@ -3,7 +3,7 @@ global: evaluation_interval: 15s scrape_configs: - - job_name: "myapp" + - job_name: "mydev" metrics_path: /metrics scheme: https # Render apps use HTTPS static_configs: From a797a9aff227f33e900ae7d13dca30e7c8a15836 Mon Sep 17 00:00:00 2001 From: shaibuuneks Date: Fri, 19 Sep 2025 11:21:41 +0000 Subject: [PATCH 8/8] Fix prometheus config with https + new job name --- infra/prometheus/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infra/prometheus/Dockerfile b/infra/prometheus/Dockerfile index fa15349..e0a6382 100644 --- a/infra/prometheus/Dockerfile +++ b/infra/prometheus/Dockerfile @@ -2,3 +2,5 @@ FROM prom/prometheus:v2.53.0 # Copy custom config COPY prometheus.yml /etc/prometheus/prometheus.yml +# force rebuild +ARG CACHEBUST=1 \ No newline at end of file