Add opentelemetry to Armada

README.md

@@ -186,6 +186,9 @@ Goreman exposes services on the following ports:
 | Redis                      | 6379  | Cache & events      |
 | PostgreSQL                 | 5432  | Database            |
 | Pulsar                     | 6650  | Message broker      |
+| OTEL Collector gRPC        | 4317  | OTLP ingest         |
+| OTEL Collector HTTP        | 4318  | OTLP ingest         |
+| Jaeger UI                  | 16686 | Trace visualization |
 
 ## Documentation
 

_local/binoculars/config-auth.yaml

@@ -1,6 +1,14 @@
 httpPort: 8084
 grpcPort: 50053
 metricsPort: 9007
+observability:
+  enabled: true
+  exporter:
+    endpoint: "http://localhost:4318"
+    protocol: "http/protobuf"
+  traces:
+    sampler: "parent_based_trace_id_ratio"
+    samplerArg: 1.0
 application:
   clusterId: local-cluster
 kubernetes:
@@ -24,4 +32,4 @@ auth:
     cancel_any_jobs: ["admins"]
     reprioritize_any_jobs: ["admins"]
     watch_all_events: ["admins"]
-    execute_jobs: ["admins", "executors"]
+    execute_jobs: ["admins", "executors"]

_local/binoculars/config.yaml

@@ -1,6 +1,14 @@
 httpPort: 8084
 grpcPort: 50053
 metricsPort: 9007
+observability:
+  enabled: true
+  exporter:
+    endpoint: "http://localhost:4318"
+    protocol: "http/protobuf"
+  traces:
+    sampler: "parent_based_trace_id_ratio"
+    samplerArg: 1.0
 application:
   clusterId: local-cluster
 kubernetes:

_local/compose/full.yaml

@@ -75,7 +75,8 @@ services:
       PULSAR_PREFIX_allowAutoTopicCreation: "true"
       PULSAR_PREFIX_allowAutoTopicCreationType: non-partitioned
       PULSAR_PREFIX_autoSkipNonRecoverableData: "true"
-    entrypoint: sh -c "bin/apply-config-from-env.py conf/standalone.conf && bin/pulsar standalone"
+    entrypoint: sh -c "bin/apply-config-from-env.py conf/standalone.conf &&
+      bin/pulsar standalone"
     ports:
       - "6650:6650"
       - "8090:8080"
@@ -88,6 +89,28 @@ services:
       retries: 10
       start_period: 30s
 
+  jaeger:
+    container_name: jaeger
+    image: ${JAEGER_IMAGE:-jaegertracing/all-in-one:1.76.0}
+    environment:
+      - COLLECTOR_OTLP_ENABLED=true
+    ports:
+      - "16686:16686" # Jaeger UI
+    restart: unless-stopped
+
+  otel-collector:
+    container_name: otel-collector
+    image: ${OTEL_IMAGE:-otel/opentelemetry-collector-contrib:0.154.0}
+    command: ["--config=/etc/otelcol-contrib/config.yaml"]
+    volumes:
+      - ../otel/collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
+    ports:
+      - "4317:4317" # OTLP gRPC
+      - "4318:4318" # OTLP HTTP
+    restart: unless-stopped
+    depends_on:
+      - jaeger
+
   # =========================================================
   # Database migrations (run once, then exit)
   # =========================================================
@@ -96,7 +119,7 @@ services:
     container_name: scheduler-migration
     image: ${ARMADA_IMAGE:-gresearch/armada-bundle}:${ARMADA_IMAGE_TAG:-latest}
     depends_on:
-      postgres: { condition: service_healthy }
+      postgres: {condition: service_healthy}
     volumes:
       - ../scheduler/config.yaml:/config/config.yaml:ro
     environment:
@@ -108,7 +131,7 @@ services:
     container_name: lookout-migration
     image: ${ARMADA_IMAGE:-gresearch/armada-lookout-bundle}:${ARMADA_IMAGE_TAG:-latest}
     depends_on:
-      postgres: { condition: service_healthy }
+      postgres: {condition: service_healthy}
     volumes:
       - ../lookout/config.yaml:/config/config.yaml:ro
     environment:
@@ -123,7 +146,7 @@ services:
     container_name: scheduler
     image: ${ARMADA_IMAGE:-gresearch/armada-bundle}:${ARMADA_IMAGE_TAG:-latest}
     depends_on:
-      scheduler-migration: { condition: service_completed_successfully }
+      scheduler-migration: {condition: service_completed_successfully}
     volumes:
       - ../scheduler/config.yaml:/config/config.yaml:ro
     environment:
@@ -140,8 +163,8 @@ services:
     container_name: scheduleringester
     image: ${ARMADA_IMAGE:-gresearch/armada-bundle}:${ARMADA_IMAGE_TAG:-latest}
     depends_on:
-      scheduler-migration: { condition: service_completed_successfully }
-      pulsar: { condition: service_healthy }
+      scheduler-migration: {condition: service_completed_successfully}
+      pulsar: {condition: service_healthy}
     volumes:
       - ../scheduleringester/config.yaml:/config/config.yaml:ro
     environment:
@@ -161,9 +184,9 @@ services:
       - "50051:50051"
       - "8081:8081"
     depends_on:
-      lookout-migration: { condition: service_completed_successfully }
-      pulsar: { condition: service_healthy }
-      redis: { condition: service_healthy }
+      lookout-migration: {condition: service_completed_successfully}
+      pulsar: {condition: service_healthy}
+      redis: {condition: service_healthy}
     volumes:
       - ../server/config.yaml:/config/config.yaml:ro
     environment:
@@ -182,7 +205,7 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
     depends_on:
-      scheduler: { condition: service_started }
+      scheduler: {condition: service_started}
     volumes:
       - ../executor/config.yaml:/config/config.yaml:ro
       - ../../.kube/internal:/.kube:ro
@@ -196,8 +219,8 @@ services:
     container_name: eventingester
     image: ${ARMADA_IMAGE:-gresearch/armada-bundle}:${ARMADA_IMAGE_TAG:-latest}
     depends_on:
-      pulsar: { condition: service_healthy }
-      redis: { condition: service_healthy }
+      pulsar: {condition: service_healthy}
+      redis: {condition: service_healthy}
     volumes:
       - ../eventingester/config.yaml:/config/config.yaml:ro
     environment:
@@ -211,8 +234,8 @@ services:
     container_name: lookoutingester
     image: ${ARMADA_IMAGE:-gresearch/armada-lookout-bundle}:${ARMADA_IMAGE_TAG:-latest}
     depends_on:
-      lookout-migration: { condition: service_completed_successfully }
-      pulsar: { condition: service_healthy }
+      lookout-migration: {condition: service_completed_successfully}
+      pulsar: {condition: service_healthy}
     volumes:
       - ../lookoutingester/config.yaml:/config/config.yaml:ro
     environment:
@@ -229,7 +252,7 @@ services:
     ports:
       - "8089:8089"
     depends_on:
-      lookout-migration: { condition: service_completed_successfully }
+      lookout-migration: {condition: service_completed_successfully}
     volumes:
       - ../lookout/config.yaml:/config/config.yaml:ro
     environment:

_local/compose/stack.yaml

@@ -61,7 +61,8 @@ services:
       PULSAR_PREFIX_allowAutoTopicCreation: "true"
       PULSAR_PREFIX_allowAutoTopicCreationType: non-partitioned
       PULSAR_PREFIX_autoSkipNonRecoverableData: "true"
-    entrypoint: sh -c "bin/apply-config-from-env.py conf/standalone.conf && bin/pulsar standalone"
+    entrypoint: sh -c "bin/apply-config-from-env.py conf/standalone.conf &&
+      bin/pulsar standalone"
     ports:
       - "6650:6650"
       - "8090:8080"
@@ -94,3 +95,25 @@ services:
     # No healthcheck on purpose: keycloak's realm import + warmup is slower than
     # `docker compose up --wait` is willing to block for. `mage dev:up auth` polls the
     # realm endpoint via waitForKeycloak before starting goreman.
+
+  jaeger:
+    container_name: jaeger
+    image: ${JAEGER_IMAGE:-jaegertracing/all-in-one:1.76.0}
+    environment:
+      - COLLECTOR_OTLP_ENABLED=true
+    ports:
+      - "16686:16686" # Jaeger UI
+    restart: unless-stopped
+
+  otel-collector:
+    container_name: otel-collector
+    image: ${OTEL_IMAGE:-otel/opentelemetry-collector-contrib:0.154.0}
+    command: ["--config=/etc/otelcol-contrib/config.yaml"]
+    volumes:
+      - ../otel/collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro
+    ports:
+      - "4317:4317" # OTLP gRPC
+      - "4318:4318" # OTLP HTTP
+    restart: unless-stopped
+    depends_on:
+      - jaeger

_local/eventingester/config.yaml

@@ -9,6 +9,14 @@ pulsar:
   jobsetEventsTopic: "events"
 subscriptionName: "events-ingester"
 metricsPort: 9004
+observability:
+  enabled: true
+  exporter:
+    endpoint: "http://localhost:4318"
+    protocol: "http/protobuf"
+  traces:
+    sampler: "parent_based_trace_id_ratio"
+    samplerArg: 1.0
 metrics:
   redis:
     enabled: true

_local/executor/config-auth.yaml

@@ -7,6 +7,14 @@ executorApiConnection:
     clientId: "armada-executor"
     clientSecret: "executor-secret"
     scopes: ["profile", "email"]
+observability:
+  enabled: true
+  exporter:
+    endpoint: "http://localhost:4318"
+    protocol: "http/protobuf"
+  traces:
+    sampler: "parent_based_trace_id_ratio"
+    samplerArg: 1.0
 metric:
   port: 9002
 application:

_local/executor/config.yaml

@@ -2,6 +2,14 @@ httpPort: 8082
 executorApiConnection:
   armadaUrl: "localhost:50052"
   forceNoTls: true
+observability:
+  enabled: true
+  exporter:
+    endpoint: "http://localhost:4318"
+    protocol: "http/protobuf"
+  traces:
+    sampler: "parent_based_trace_id_ratio"
+    samplerArg: 1.0
 metric:
   port: 9002
 application:

_local/lookout/config-auth.yaml

@@ -1,5 +1,13 @@
 apiPort: 8089
 metricsPort: 9003
+observability:
+  enabled: true
+  exporter:
+    endpoint: "http://localhost:4318"
+    protocol: "http/protobuf"
+  traces:
+    sampler: "parent_based_trace_id_ratio"
+    samplerArg: 1.0
 corsAllowedOrigins:
   - "http://localhost:3000"
   - "http://localhost:8089"

_local/lookout/config.yaml

@@ -1,5 +1,13 @@
 apiPort: 8089
 metricsPort: 9003
+observability:
+  enabled: true
+  exporter:
+    endpoint: "http://localhost:4318"
+    protocol: "http/protobuf"
+  traces:
+    sampler: "parent_based_trace_id_ratio"
+    samplerArg: 1.0
 corsAllowedOrigins:
   - "http://localhost:3000"
   - "http://localhost:8089"

_local/lookoutingester/config.yaml

@@ -1,4 +1,12 @@
 metricsPort: 9005
+observability:
+  enabled: true
+  exporter:
+    endpoint: "http://localhost:4318"
+    protocol: "http/protobuf"
+  traces:
+    sampler: "parent_based_trace_id_ratio"
+    samplerArg: 1.0
 postgres:
   connection:
     host: localhost

_local/otel/collector-config.yaml

@@ -0,0 +1,23 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  batch:
+
+exporters:
+  otlp/jaeger:
+    endpoint: jaeger:4317
+    tls:
+      insecure: true
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp/jaeger]

_local/scheduler/config-auth.yaml

@@ -2,6 +2,14 @@ grpc:
   port: 50052
   tls:
     enabled: false
+observability:
+  enabled: true
+  exporter:
+    endpoint: "http://localhost:4318"
+    protocol: "http/protobuf"
+  traces:
+    sampler: "parent_based_trace_id_ratio"
+    samplerArg: 1.0
 auth:
   anonymousAuth: true
   permissionGroupMapping:

_local/scheduler/config.yaml

@@ -2,6 +2,14 @@ grpc:
   port: 50052
   tls:
     enabled: false
+observability:
+  enabled: true
+  exporter:
+    endpoint: "http://localhost:4318"
+    protocol: "http/protobuf"
+  traces:
+    sampler: "parent_based_trace_id_ratio"
+    samplerArg: 1.0
 auth:
   anonymousAuth: true
   permissionGroupMapping:

_local/scheduleringester/config.yaml

@@ -1,5 +1,13 @@
 metrics:
   port: 9006
+observability:
+  enabled: true
+  exporter:
+    endpoint: "http://localhost:4318"
+    protocol: "http/protobuf"
+  traces:
+    sampler: "parent_based_trace_id_ratio"
+    samplerArg: 1.0
 postgres:
   connection:
     host: localhost

_local/server/config-auth.yaml

@@ -1,6 +1,14 @@
 httpPort: 8081
 grpcPort: 50051
 metricsPort: 9000
+observability:
+  enabled: true
+  exporter:
+    endpoint: "http://localhost:4318"
+    protocol: "http/protobuf"
+  traces:
+    sampler: "parent_based_trace_id_ratio"
+    samplerArg: 1.0
 corsAllowedOrigins:
   - "http://localhost:3000"
   - "http://localhost:8089"

_local/server/config.yaml

@@ -1,6 +1,14 @@
 httpPort: 8081
 grpcPort: 50051
 metricsPort: 9000
+observability:
+  enabled: true
+  exporter:
+    endpoint: "http://localhost:4318"
+    protocol: "http/protobuf"
+  traces:
+    sampler: "parent_based_trace_id_ratio"
+    samplerArg: 1.0
 corsAllowedOrigins:
   - "http://localhost:3000"
   - "http://localhost:8089"