feat: basic otel & jaeger tracing

.github/workflows/infrastructure.yaml

@@ -107,6 +107,7 @@ jobs:
 
       - name: Test Container reachability
         run: |
+          sleep 10
           GATEWAY_IP=$(kubectl -n istio-gateway get svc istio-gateway -o jsonpath='{.status.loadBalancer.ingress[].ip}')
           curl --fail http://dev.local/api/time --resolve "dev.local:80:${GATEWAY_IP}"
 

Tiltfile

@@ -1,3 +1,4 @@
+update_settings(max_parallel_updates = 3, k8s_upsert_timeout_secs = 300, suppress_unused_image_warnings = None)
 # install required cluster tooling
 include('./local-dev/Tiltfile')
 
@@ -9,22 +10,24 @@ k8s_resource(
   workload = 'frontend',
   labels = ['App'],
   port_forwards = '8080:8080',
-  resource_deps = ['istio-gateway'],
+  resource_deps = ['istio-gateway', 'otel-collector'],
 )
 k8s_resource(
   workload = 'backend',
   labels = ['App'],
   port_forwards = '8081:8080',
-  resource_deps = ['istio-gateway'],
+  resource_deps = ['istio-gateway', 'otel-collector'],
 )
 
 k8s_resource(
   objects=[
     'virtualservice:virtualservice',
     'strict-mtls:peerauthentication',
     'isolation:authorizationpolicy',
+    'instrumentation:instrumentation',
+    'otel:telemetry',
   ],
   new_name='istio-config',
   labels = ['App'],
-  resource_deps = ['istio-gateway'],
+  resource_deps = ['istio-gateway', 'wait-otel-operator-ready'],
 )

backend/Tiltfile

@@ -1,14 +1,14 @@
 github_sha = os.environ.get("GITHUB_SHA", "local")[:7]
 
 docker_build(
-  'localhost:5001/gitops-demo/backend',
+  'localhost:5001/gitops-playground/backend',
   '.',
   only = [
     'Dockerfile',
     'go.mod',
     'main.go',
   ],
     extra_tag = [
-      'localhost:5001/gitops-demo/backend:ci-%s' % github_sha
+      'localhost:5001/gitops-playground/backend:ci-%s' % github_sha
     ]
 )

frontend/Tiltfile

@@ -1,7 +1,7 @@
 github_sha = os.environ.get("GITHUB_SHA", "local")[:7]
 
 docker_build(
-  'localhost:5001/gitops-demo/frontend',
+  'localhost:5001/gitops-playground/frontend',
   '.',
   only = [
     'Dockerfile',
@@ -11,6 +11,6 @@ docker_build(
     'template'
   ],
   extra_tag = [
-    'localhost:5001/gitops-demo/frontend:ci-%s' % github_sha
+    'localhost:5001/gitops-playground/frontend:ci-%s' % github_sha
   ]
 )

local-dev/Tiltfile

@@ -12,3 +12,6 @@ include('./kiali/Tiltfile')
 
 # Install Prometheus
 include('./prometheus/Tiltfile')
+
+# Install OpenTelemetry Collector
+include('./otel-collector/Tiltfile')

local-dev/istio/Tiltfile

@@ -37,7 +37,10 @@ helm_resource(
     '--set=meshConfig.enableTracing=true',
     '--set=meshConfig.ingressService=istio-gateway',
     '--set=meshConfig.ingressSelector=gateway',
-    '--set=meshConfig.defaultConfig.tracing.zipkin.address=jaeger-collector.istio-system:9411',
+    '--set=meshConfig.defaultConfig.tracing.zipkin.address=jaeger.observability.svc.cluster.local:9411',
+    '--set=meshconfig.extensionProviders[0].name=otel-tracing',
+    '--set=meshconfig.extensionProviders[0].opentelemetry.port=4317',
+    '--set=meshconfig.extensionProviders[0].opentelemetry.service=otel-collector-collector.observability.svc.cluster.local',
     '--wait',
   ],
   auto_init = True
@@ -61,7 +64,7 @@ helm_resource(
 k8s_yaml('gateway.yaml')
 k8s_resource(
   new_name = 'Gateway',
-  objects = ['tls-gateway'],
+  objects = ['kubedev-net'],
   resource_deps = ['istio-gateway'],
   labels = ['System'],
 )

local-dev/istio/gateway.yaml

@@ -1,7 +1,7 @@
 apiVersion: networking.istio.io/v1beta1
 kind: Gateway
 metadata:
-  name: tls-gateway
+  name: kubedev-net
   namespace: istio-gateway
 spec:
   selector:

local-dev/jaeger/Tiltfile

@@ -9,10 +9,10 @@ helm_resource(
   'jaeger',
   'jaegertracing/jaeger',
   resource_deps = ['istio'],
-  namespace = 'istio-system',
+  namespace = 'observability',
   labels = 'System',
   flags = [
-    '--version=3.4.1',
+    '--version=4.3.4',
     '--create-namespace',
     '--set=query.enabled=false',
     '--set=provisionDataStore.cassandra=false',
@@ -22,7 +22,6 @@ helm_resource(
     '--set=collector.enabled=false',
   ],
   auto_init = False,
-  pod_readiness = 'ignore'
 )
 
 k8s_resource(

local-dev/kiali/Tiltfile

@@ -9,18 +9,18 @@ helm_resource(
   'kiali',
   'kiali-server/kiali-server',
   resource_deps = ['istio', 'jaeger', 'prometheus'],
-  namespace = 'istio-system',
+  namespace = 'observability',
   labels = 'System',
   flags = [
     '--version=2.19.0',
     '--create-namespace',
     '--set=clustering.autodetect_secrets.enabled=false',
     '--set=kiali_feature_flags.validations.skip_wildcard_gateway_hosts=true',
     '--set=external_services.prometheus.enabled=true',
-    '--set=external_services.prometheus.url=http://prometheus-server.prometheus',
+    '--set=external_services.prometheus.url=http://prometheus-server',
     '--set=external_services.grafana.enabled=false',
     '--set=external_services.tracing.enabled=true',
-    '--set=external_services.tracing.internal_url=http://jaeger-query:16685/jaeger',
+    '--set=external_services.tracing.internal_url=http://jaeger:16685/jaeger',
     '--set=external_services.tracing.use_grpc=true',
     '--set=istio.root_namespace=istio-system',
     '--set=istio.component_status.enabled=true',
@@ -32,7 +32,6 @@ helm_resource(
     '--set=istio.component_status.components[1].namespace=istio-gateway',
   ],
   auto_init = False,
-  pod_readiness = 'ignore'
 )
 
 k8s_resource(

local-dev/kind-with-registry.sh

@@ -25,7 +25,7 @@ kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
 - role: control-plane
-  image: kindest/node:v1.33.4@sha256:25a6018e48dfcaee478f4a59af81157a437f15e6e140bf103f85a2e7cd0cbbf2
+  image: kindest/node:v1.34.3@sha256:08497ee19eace7b4b5348db5c6a1591d7752b164530a36f855cb0f2bdcbadd48
 containerdConfigPatches:
 - |-
   [plugins."io.containerd.grpc.v1.cri".registry]

local-dev/otel-collector/Tiltfile

@@ -0,0 +1,47 @@
+load('ext://helm_resource', 'helm_repo', 'helm_resource')
+
+helm_repo(
+  'opentelemetry-repo',
+  'https://open-telemetry.github.io/opentelemetry-helm-charts'
+)
+
+helm_resource(
+  'opentelemetry',
+  'opentelemetry-repo/opentelemetry-operator',
+  #resource_deps = ['istio'],
+  namespace = 'observability',
+  labels = 'System',
+  flags = [
+    '--version=0.102.0',
+    '--create-namespace',
+    '--set=manager.autoInstrumentation.go.enabled=true',
+    '--set=admissionWebhooks.certManager.enabled=false',
+  ],
+  auto_init = True,
+)
+
+k8s_resource(
+  workload = 'opentelemetry',
+  labels = ['System'],
+)
+
+local_resource(
+  name='wait-otel-operator-ready',
+  cmd='''
+set -e
+
+kubectl wait --for=condition=Established --timeout=180s crd/opentelemetrycollectors.opentelemetry.io
+kubectl wait --for=condition=Established --timeout=180s crd/instrumentations.opentelemetry.io
+
+kubectl -n observability rollout status deploy/opentelemetry-opentelemetry-operator --timeout=180s && sleep 5
+''',
+  resource_deps=['opentelemetry'],
+)
+
+k8s_yaml('otel-collector.yaml')
+k8s_resource(
+  new_name = 'otel-collector',
+  objects = ['otel-collector'],
+  resource_deps = ['wait-otel-operator-ready'],
+  labels = ['System'],
+)

local-dev/otel-collector/otel-collector.yaml

@@ -0,0 +1,48 @@
+apiVersion: opentelemetry.io/v1beta1
+kind: OpenTelemetryCollector
+metadata:
+  name: otel-collector
+  namespace: observability
+spec:
+  mode: daemonset
+  config:
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:4317
+          http:
+            endpoint: 0.0.0.0:4318
+    processors:
+      batch:
+        timeout: 10s
+        send_batch_size: 1024
+    exporters:
+      debug:
+        verbosity: detailed
+      otlphttp/prometheus:
+        endpoint: http://prometheus-server:9090/api/v1/otlp
+        tls:
+          insecure: true
+      otlp:
+        endpoint: jaeger:4317
+        tls:
+          insecure: true
+        sending_queue:
+          enabled: true
+        retry_on_failure:
+          enabled: true
+    service:
+      pipelines:
+        logs:
+          receivers: [otlp]
+          processors: [batch]
+          exporters: [debug]
+        traces:
+          receivers: [otlp]
+          processors: [batch]
+          exporters: [debug, otlp]
+        metrics:
+          receivers: [otlp]
+          processors: [batch]
+          exporters: [debug, otlphttp/prometheus]

local-dev/prometheus/Tiltfile

@@ -8,7 +8,7 @@ helm_repo(
 helm_resource(
   'prometheus',
   'prometheus-community/prometheus',
-  namespace='prometheus',
+  namespace='observability',
   labels = 'System',
   flags = [
     '--version=27.52.0',

manifests/base/kustomization.yaml

@@ -6,6 +6,7 @@ resources:
 - backend
 - isolation.yaml
 - peer-authentication.yaml
+- otel-instrumentation.yaml
 
 replacements:
   - source:

manifests/base/otel-instrumentation.yaml

@@ -0,0 +1,23 @@
+apiVersion: opentelemetry.io/v1alpha1
+kind: Instrumentation
+metadata:
+  name: instrumentation
+spec:
+  exporter:
+    endpoint: http://otel-collector-collector.observability.svc.cluster.local:4318
+  propagators:
+    - tracecontext
+    - baggage
+  sampler:
+    type: parentbased_traceidratio
+    argument: "1"
+---
+apiVersion: telemetry.istio.io/v1
+kind: Telemetry
+metadata:
+  name: otel
+spec:
+  tracing:
+    - providers:
+        - name: otel-tracing
+      randomSamplingPercentage: 100

manifests/stages/local-dev/kustomization.yaml

@@ -28,8 +28,8 @@ configMapGenerator:
       - BACKEND_URL=http://backend
 
 images:
-  - name: ghcr.io/procinger/gitops-demo-frontend/frontend
-    newName: localhost:5001/gitops-demo/frontend
+  - name: ghcr.io/procinger/gitops-playground/frontend
+    newName: localhost:5001/gitops-playground/frontend
 
-  - name: ghcr.io/procinger/gitops-demo-frontend/backend
-    newName: localhost:5001/gitops-demo/backend
+  - name: ghcr.io/procinger/gitops-playground/backend
+    newName: localhost:5001/gitops-playground/backend