From 6777d46f5d087ddbfbfab450a19c5ba2aa179b8a Mon Sep 17 00:00:00 2001 From: Keith Ralphs Date: Tue, 17 Feb 2026 13:33:12 +0000 Subject: [PATCH 1/4] Disable health tracing and endpoints during debug --- helm/blueapi/README.md | 2 +- helm/blueapi/templates/configmap.yaml | 2 ++ helm/blueapi/templates/statefulset.yaml | 7 ++++++- helm/blueapi/values.schema.json | 8 ++++++++ helm/blueapi/values.yaml | 3 +++ 5 files changed, 20 insertions(+), 2 deletions(-) diff --git a/helm/blueapi/README.md b/helm/blueapi/README.md index e4fe2c162a..771c483fcc 100644 --- a/helm/blueapi/README.md +++ b/helm/blueapi/README.md @@ -44,7 +44,7 @@ A Helm chart deploying a worker pod that runs Bluesky plans | serviceAccount.name | string | `""` | | | startupProbe | object | `{"failureThreshold":5,"httpGet":{"path":"/healthz","port":"http"},"periodSeconds":10}` | A more lenient livenessProbe to allow the service to start fully. This is automatically disabled when in debug mode. | | tolerations | list | `[]` | May be required to run on specific nodes (e.g. the control machine) | -| tracing | object | `{"otlp":{"enabled":false,"protocol":"http/protobuf","server":{"host":"http://opentelemetry-collector.tracing","port":4318}}}` | Configure tracing: opentelemetry-collector.tracing should be available in all Diamond clusters | +| tracing | object | `{"fastapi":{"excludedURLs":"/healthz"},"otlp":{"enabled":false,"protocol":"http/protobuf","server":{"host":"http://opentelemetry-collector.tracing","port":4318}}}` | Exclude health probe requests from tracing by default to prevent spamming | | volumeMounts | list | `[{"mountPath":"/config","name":"worker-config","readOnly":true}]` | Additional volumeMounts on the output StatefulSet definition. Define how volumes are mounted to the container referenced by using the same name. | | volumes | list | `[]` | Additional volumes on the output StatefulSet definition. Define volumes from e.g. Secrets, ConfigMaps or the Filesystem | | worker | object | `{"api":{"url":"http://0.0.0.0:8000/"},"env":{"sources":[{"kind":"planFunctions","module":"dodal.plans"},{"kind":"planFunctions","module":"dodal.plan_stubs.wrapped"}]},"logging":{"graylog":{"enabled":false,"url":"tcp://graylog-log-target.diamond.ac.uk:12231/"},"level":"INFO"},"scratch":{"repositories":[],"root":"/workspace"},"stomp":{"auth":{"password":"guest","username":"guest"},"enabled":false,"url":"tcp://rabbitmq:61613/"}}` | Config for the worker goes here, will be mounted into a config file | diff --git a/helm/blueapi/templates/configmap.yaml b/helm/blueapi/templates/configmap.yaml index d22583bff1..d9204bec9d 100644 --- a/helm/blueapi/templates/configmap.yaml +++ b/helm/blueapi/templates/configmap.yaml @@ -19,6 +19,8 @@ data: OTEL_EXPORTER_OTLP_ENDPOINT: {{ required "OTLP export enabled but server address not set" .Values.tracing.otlp.server.host }}:{{ .Values.tracing.otlp.server.port | default 4318 }} {{ end }} + OTEL_PYTHON_FASTAPI_EXCLUDED_URLS: {{ .Values.tracing.fastapi.excludedURLs | default "" }} + --- {{- if .Values.initContainer.enabled }} diff --git a/helm/blueapi/templates/statefulset.yaml b/helm/blueapi/templates/statefulset.yaml index a782cde599..3811629aba 100644 --- a/helm/blueapi/templates/statefulset.yaml +++ b/helm/blueapi/templates/statefulset.yaml @@ -182,7 +182,7 @@ spec: - "true" - "-m" - "blueapi" - {{- end }} + {{- else }} {{- with .Values.livenessProbe }} livenessProbe: {{- toYaml . | nindent 12 }} @@ -195,12 +195,17 @@ spec: startupProbe: {{- toYaml . | nindent 12 }} {{- end }} + {{- end }} envFrom: - configMapRef: name: {{ include "blueapi.fullname" . }}-otel-config env: - name: UV_CACHE_DIR value: {{ (.Values.worker.scratch).root }}/.uv-cache + {{- if .Values.debug.enabled }} + - name: DEBUG_MODE + value: ON + {{- end }} {{- if .Values.extraEnvVars }} {{- toYaml .Values.extraEnvVars | nindent 12 }} {{- end }} diff --git a/helm/blueapi/values.schema.json b/helm/blueapi/values.schema.json index 1578c0dad8..994ed66515 100644 --- a/helm/blueapi/values.schema.json +++ b/helm/blueapi/values.schema.json @@ -296,6 +296,14 @@ "description": "Configure tracing: opentelemetry-collector.tracing should be available in all Diamond clusters", "type": "object", "properties": { + "fastapi": { + "type": "object", + "properties": { + "excludedURLs": { + "type": "string" + } + } + }, "otlp": { "type": "object", "properties": { diff --git a/helm/blueapi/values.yaml b/helm/blueapi/values.yaml index ec6515921d..3c6e03fd48 100644 --- a/helm/blueapi/values.yaml +++ b/helm/blueapi/values.yaml @@ -163,6 +163,7 @@ extraEnvVars: [] # key: tiled_api_key # -- Configure tracing: opentelemetry-collector.tracing should be available in all Diamond clusters +# -- Exclude health probe requests from tracing by default to prevent spamming tracing: otlp: enabled: false @@ -170,6 +171,8 @@ tracing: server: host: http://opentelemetry-collector.tracing port: 4318 + fastapi: + excludedURLs: "/healthz" # @schema $ref: config_schema.json # -- Config for the worker goes here, will be mounted into a config file From 4e637da9401473d98cd6be1a11fb3ea275f5066a Mon Sep 17 00:00:00 2001 From: Keith Ralphs Date: Wed, 18 Feb 2026 15:00:58 +0000 Subject: [PATCH 2/4] change default --- helm/blueapi/templates/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/blueapi/templates/configmap.yaml b/helm/blueapi/templates/configmap.yaml index d9204bec9d..67dd18e848 100644 --- a/helm/blueapi/templates/configmap.yaml +++ b/helm/blueapi/templates/configmap.yaml @@ -19,7 +19,7 @@ data: OTEL_EXPORTER_OTLP_ENDPOINT: {{ required "OTLP export enabled but server address not set" .Values.tracing.otlp.server.host }}:{{ .Values.tracing.otlp.server.port | default 4318 }} {{ end }} - OTEL_PYTHON_FASTAPI_EXCLUDED_URLS: {{ .Values.tracing.fastapi.excludedURLs | default "" }} + OTEL_PYTHON_FASTAPI_EXCLUDED_URLS: {{ .Values.tracing.fastapi.excludedURLs | default "/healthz" }} --- From 8b2831fed05fd0ccdc73e16852ee64735001b8fc Mon Sep 17 00:00:00 2001 From: Keith Ralphs Date: Wed, 18 Feb 2026 15:35:38 +0000 Subject: [PATCH 3/4] correct DEBUG_ON env value --- helm/blueapi/templates/configmap.yaml | 3 ++- helm/blueapi/templates/statefulset.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/helm/blueapi/templates/configmap.yaml b/helm/blueapi/templates/configmap.yaml index 67dd18e848..6551104a3e 100644 --- a/helm/blueapi/templates/configmap.yaml +++ b/helm/blueapi/templates/configmap.yaml @@ -17,9 +17,10 @@ data: OTLP_EXPORT_ENABLED: "true" OTEL_EXPORTER_OTLP_TRACES_PROTOCOL: {{ .Values.tracing.otlp.protocol | default "http/protobuf" }} OTEL_EXPORTER_OTLP_ENDPOINT: {{ required "OTLP export enabled but server address not set" .Values.tracing.otlp.server.host }}:{{ .Values.tracing.otlp.server.port | default 4318 }} + OTEL_PYTHON_FASTAPI_EXCLUDED_URLS: {{ .Values.tracing.fastapi.excludedURLs | default "" }} {{ end }} - OTEL_PYTHON_FASTAPI_EXCLUDED_URLS: {{ .Values.tracing.fastapi.excludedURLs | default "/healthz" }} + --- diff --git a/helm/blueapi/templates/statefulset.yaml b/helm/blueapi/templates/statefulset.yaml index 3811629aba..2f9db1c81c 100644 --- a/helm/blueapi/templates/statefulset.yaml +++ b/helm/blueapi/templates/statefulset.yaml @@ -204,7 +204,7 @@ spec: value: {{ (.Values.worker.scratch).root }}/.uv-cache {{- if .Values.debug.enabled }} - name: DEBUG_MODE - value: ON + value: "ON" {{- end }} {{- if .Values.extraEnvVars }} {{- toYaml .Values.extraEnvVars | nindent 12 }} From 5e565f09924bb083823e2dfe6c423cae8d539b9a Mon Sep 17 00:00:00 2001 From: Keith Ralphs Date: Wed, 18 Feb 2026 16:42:44 +0000 Subject: [PATCH 4/4] remove blank lines --- helm/blueapi/templates/configmap.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/helm/blueapi/templates/configmap.yaml b/helm/blueapi/templates/configmap.yaml index 6551104a3e..aa813e6485 100644 --- a/helm/blueapi/templates/configmap.yaml +++ b/helm/blueapi/templates/configmap.yaml @@ -20,8 +20,6 @@ data: OTEL_PYTHON_FASTAPI_EXCLUDED_URLS: {{ .Values.tracing.fastapi.excludedURLs | default "" }} {{ end }} - - --- {{- if .Values.initContainer.enabled }}