osparc-ops-environments/services/monitoring/prometheus/prometheus-base.yml at 0decbe042e6001875af4c4a21949ae380fea5779 · ITISFoundation/osparc-ops-environments · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
#  global config
#  DOLLAR SIGNS NEED TO BE EXCAPED (see https://stackoverflow.com/a/61259844/10198629)
global:
  scrape_interval:  ${PROMETHEUS_SCRAPE_INTERVAL}s
  evaluation_interval: ${PROMETHEUS_SCRAPE_INTERVAL}s # By default, scrape targets every 15 seconds.
  scrape_timeout: ${PROMETHEUS_SCRAPE_TIMEOUT}s

  # Attach these labels to any time series or alerts when communicating with
  # external systems (federation, remote storage, Alertmanager).
  external_labels:
    monitor: "simcore-monitor"

# Load and evaluate rules in this file every 'evaluation_interval' seconds.
rule_files:
  - "alert.rules"
  - "prometheus.rules.yml"

# alert
alerting:
  alertmanagers:
    - scheme: http
      static_configs:
        - targets:
            - "alertmanager:9093"

# A scrape configuration containing exactly one endpoint to scrape:
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.

  # Create a job for Docker daemons. see [https://prometheus.io/docs/guides/dockerswarm/]
  - job_name: "docker-engines"
    dockerswarm_sd_configs:
      - host: unix:///var/run/docker.sock
        role: nodes # this scrapes docker nodes
    relabel_configs:
      # Fetch metrics on port 9323.
      - source_labels: [__meta_dockerswarm_node_address]
        target_label: __address__
        replacement: $${empty_var}1:9323
      - source_labels: [__meta_dockerswarm_node_hostname]
        target_label: instance
      # Set hostname as instance label
      # Swarm manager give the address 0.0.0.0 and is then not scraped by Prometheus. Fixed thansk to https://github.com/prometheus/prometheus/issues/11060#issuecomment-1195278301
      - source_labels:
          - __meta_dockerswarm_node_manager_leader
          - __meta_dockerswarm_node_manager_address
        regex: 'true;(.+):[0-9]+'
        target_label: __address__
        replacement: $${empty_var}1:9323

  # Create a job for Docker Swarm containers.
  # Prometheus docker swarm discovery will automatically discover services that need to be scraped by prometheus
  # All services that need to be monitored must at least define the following SERVICE labels (in deploy/labels section):
  # deploy:
  #   labels:
  #     # prometheus labels
  #     - prometheus-job=traefik_ops
  #     - prometheus-port=8082
  #
  #
  # Swarm tasks prometheus service discovery:
  # There is one task discovered per network that the containe is associated to.
  - job_name: "dockerswarm-tasks"
    dockerswarm_sd_configs:
      - host: unix:///var/run/docker.sock
        role: tasks # this scrapes docker tasks
    relabel_configs:
      # keep only metrics that are available over monitored network
      - source_labels: [__meta_dockerswarm_network_name]
        regex: ${MONITORED_NETWORK}
        action: keep
      # Only keep containers that should be running
      - source_labels: [__meta_dockerswarm_task_desired_state]
        regex: running
        action: keep
      # Only keep containers that have a `prometheus-job` label.
      - source_labels: [__meta_dockerswarm_service_label_prometheus_job]
        regex: .+
        action: keep
      # Keep the containers IP and Port, very necessary for our setup DONT MESS WITH THIS
      - source_labels:
          [__address__, __meta_dockerswarm_service_label_prometheus_port]
        separator: ";"
        regex: "(.*):.*;(.*)"
        target_label: __address__
        replacement: $${empty_var}1:$${empty_var}2
      #
      # Use the prometheus-port Swarm label as Prometheus job port.
      - source_labels: [__meta_dockerswarm_service_label_prometheus_job]
        target_label: job
      # Set hostname as instance label
      - source_labels: [__meta_dockerswarm_node_hostname]
        target_label: instance
    # Toss container address:
      #- action: labeldrop
      #  regex: __address__
    metric_relabel_configs:
      #
      # These are WORK_IN_PROGRESS: This might in the future add the service limits and reservations as unique prometheus labels, but
      # this would require soffisticated regex parsing
      # To use these: # KEEP: container_label_simcore_service_settings
      #
      - regex: "container_label_com_docker_compose_config_hash"
        action: labeldrop  # cAdvisor pruning
      - regex: "container_label_com_docker_compose_container_number"
        action: labeldrop  # cAdvisor pruning
      - regex: "container_label_simcore_service_compose_spec"
        action: labeldrop  # cAdvisor pruning
      - regex: "container_label_simcore_service_container_http_entrypoint"
        action: labeldrop  # cAdvisor pruning
      - regex: "container_label_simcore_service_paths_mapping"
        action: labeldrop  # cAdvisor pruning
      - regex: "container_label_org_.*"
        action: labeldrop  # cAdvisor pruning
      - regex: "container_label_com_docker_compose_project"
        action: labeldrop  # cAdvisor pruning
      - regex: "container_label_com_docker_compose_project_config_files"
        action: labeldrop  # cAdvisor pruning
      - regex: "container_label_com_docker_compose_service"
        action: labeldrop  # cAdvisor pruning
      - regex: "container_label_com_docker_compose_project"
        action: labeldrop  # cAdvisor pruning
      - regex: "container_label_maintainer"
        action: labeldrop  # cAdvisor pruning
      - regex: "container_label_com_docker_compose_project_working_dir"
        action: labeldrop
      - regex: "container_label_io_simcore_contact"
        action: labeldrop
      - regex: "container_label_io_simcore_authors"
        action: labeldrop
      - regex: "container_label_io_simcore_description"
        action: labeldrop
      - regex: "container_label_io_simcore_inputs"
        action: labeldrop
      - regex: "container_label_io_simcore_key"
        action: labeldrop
      - regex: "container_label_io_simcore_outputs"
        action: labeldrop
      - regex: "container_label_io_simcore_runtime_swarm_stack_name"
        action: labeldrop
      - regex: "container_label_io_simcore_thumbnail"
        action: labeldrop
      - regex: "container_label_simcore_service_settings"
        action: labeldrop

### Used:
# container_label_com_docker_swarm_node_id
# container_label_com_docker_swarm_service_name