🎨 Add prometheus metrics: vector, loki, tempo, grafana, jaeger

services/jaeger/docker-compose.yml.j2

@@ -41,6 +41,9 @@ services:
     command:
       - "--config=/etc/otel/config.yaml"
     deploy:
+      labels:
+        - prometheus-job=otel-collector
+        - prometheus-port=8888
       placement:
         constraints:
           - node.labels.ops==true

services/jaeger/opentelemetry-collector-config.yaml

@@ -19,6 +19,14 @@ service:
       exporters: [otlphttp,otlp]
       processors: [batch,filter/drop_healthcheck]
   telemetry:
+    metrics:
+      readers:
+        - pull:
+            exporter:
+              prometheus:
+                host: '0.0.0.0'
+                port: 8888
+
     logs:
       level: ${TRACING_OPENTELEMETRY_COLLECTOR_SERVICE_TELEMETRY_LOG_LEVEL}
 processors:

services/logging/docker-compose.yml.j2

@@ -123,13 +123,15 @@ services:
         target: /etc/vector/vector.yaml
     deploy:
       replicas: 1
+      labels:
+        - prometheus-job=vector
+        - prometheus-port=9598
       resources:
         limits:
           cpus: "1.0"
           memory: 512M
         reservations:
           memory: 256M
-      labels: []
     networks:
       logging:
 
@@ -153,6 +155,9 @@ services:
       - S3_ENDPOINT_LOKI=${S3_ENDPOINT_LOKI}
       - LOKI_RETENTION_PERIOD=${LOKI_RETENTION_PERIOD}
     deploy:
+      labels:
+        - prometheus-job=loki
+        - prometheus-port=3100
       placement:
         constraints: []
       replicas: 1

services/logging/vector.yaml

@@ -2,6 +2,9 @@
 
 sources:
   # Receive GELF messages from Docker containers via UDP
+  vector_metrics:
+    type: internal_metrics
+    scrape_interval_secs: 23
   docker_gelf:
     type: socket
     address: "0.0.0.0:12201"
@@ -115,7 +118,11 @@ sinks:
 
     healthcheck:
       enabled: true
-
+  prometheus_exporter:
+    type: prometheus_exporter
+    inputs:
+      - vector_metrics
+    address: "0.0.0.0:9598"
   # Send to Graylog via GELF over TCP
   graylog:
     type: socket

services/monitoring/docker-compose.yml.j2

@@ -230,12 +230,11 @@ services:
       - monitored # needed to access postgres
       - public
     deploy:
-      #restart_policy:
-      #  condition: on-failure
       labels:
+        - prometheus-job=grafana
+        - prometheus-port=3000
         - traefik.enable=true
         - traefik.swarm.network=${PUBLIC_NETWORK}
-        # direct access through port
         - traefik.http.services.grafana.loadbalancer.server.port=3000
         - traefik.http.routers.grafana.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/grafana`)
         - traefik.http.routers.grafana.entrypoints=https
@@ -391,6 +390,8 @@ services:
       - monitored
     deploy:
       labels:
+        - prometheus-job=tempo
+        - prometheus-port=3200
         - traefik.enable=true
         - traefik.swarm.network=${PUBLIC_NETWORK}
         - traefik.http.services.tempo.loadbalancer.server.port=9095

services/monitoring/tempo_config.yaml.j2

@@ -1,4 +1,5 @@
 server:
+  http_listen_address: 0.0.0.0
   http_listen_port: 3200
 
 distributor:
@@ -70,3 +71,5 @@ overrides:
       rate_limit_bytes: 30000000
       burst_size_bytes: 40000000
       max_traces_per_user: 10000
+usage_report:
+  reporting_enabled: false