Deploy Prometheus & node-exporter

To start off, we are only scraping Prometheus itself and node-exporter.

8 files changed, 156 insertions, 0 deletions

diff --git a/host_vars/ritchie/prometheus.yml b/host_vars/ritchie/prometheus.yml
new file mode 100644
index 0000000..63cef52
--- /dev/null
+++ b/host_vars/ritchie/prometheus.yml
@@ -0,0 +1,35 @@
+---
+prometheus_configuration:
+  global:
+    scrape_interval: 15s  # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+    evaluation_interval: 15s  # Evaluate rules every 15 seconds. The default is every 1 minute.
+    # scrape_timeout is set to the global default (10s).
+
+  # Alertmanager configuration
+  alerting:
+    alertmanagers:
+      - static_configs:
+          - targets: []
+
+  rule_files:
+    # - "first_rules.yml"
+    # - "second_rules.yml"
+
+  scrape_configs:
+    # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+    - job_name: prometheus
+
+      # Override the global default and scrape targets from this job every 5 seconds.
+      scrape_interval: 5s
+      scrape_timeout: 5s
+
+      # metrics_path defaults to '/metrics'
+      # scheme defaults to 'http'.
+
+      static_configs:
+        - targets: ['localhost:9090']
+
+    - job_name: node
+      # Scrape node exporters on all hosts
+      static_configs:
+        - targets: "{{ hostvars.values() | map(attribute='ansible_wg0.ipv4.address') | map('regex_replace', '^(.*)$', '\\1:9100') | list }}"
diff --git a/playbook.yml b/playbook.yml
index 83389f4..1ac7637 100644
--- a/playbook.yml
+++ b/playbook.yml
@@ -4,8 +4,14 @@
     - common
     - jumpcloud
     - ufw
+    - prometheus-node-exporter
     - wireguard
 
+- name: Deploy our monitoring stack
+  hosts: ritchie
+  roles:
+    - prometheus
+
 - name: Deploy nginx to hosts
   hosts: nginx
   roles:
diff --git a/roles/prometheus-node-exporter/README.md b/roles/prometheus-node-exporter/README.md
new file mode 100644
index 0000000..97ed275
--- /dev/null
+++ b/roles/prometheus-node-exporter/README.md
@@ -0,0 +1,3 @@
+# Role "prometheus-node-exporter"
+
+Installs prometheus-node-exporter on target hosts.
diff --git a/roles/prometheus-node-exporter/tasks/main.yml b/roles/prometheus-node-exporter/tasks/main.yml
new file mode 100644
index 0000000..d28eeb7
--- /dev/null
+++ b/roles/prometheus-node-exporter/tasks/main.yml
@@ -0,0 +1,7 @@
+---
+- name: install prometheus-node-exporter
+  package:
+    name: prometheus-node-exporter
+    state: present
+  tags:
+    - role::prometheus-node-exporter
diff --git a/roles/prometheus/README.md b/roles/prometheus/README.md
new file mode 100644
index 0000000..febe029
--- /dev/null
+++ b/roles/prometheus/README.md
@@ -0,0 +1,13 @@
+# Role "prometheus"
+
+Installs and configured Prometheus on target servers.
+
+
+## Variables
+
+- `prometheus_cmdline_options` configures arguments to be added
+  to the prometheus command line, and changing it will result in
+  a restart.
+
+- `prometheus_configuration` is the prometheus configuration, serialized to
+  YAML by Ansible. If unset, the default Prometheus configuration is used.
diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml
new file mode 100644
index 0000000..fbefe91
--- /dev/null
+++ b/roles/prometheus/defaults/main.yml
@@ -0,0 +1,45 @@
+---
+# Default Prometheus configuration sample
+prometheus_configuration:
+  global:
+    scrape_interval: 15s  # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+    evaluation_interval: 15s  # Evaluate rules every 15 seconds. The default is every 1 minute.
+    # scrape_timeout is set to the global default (10s).
+
+    # Attach these labels to any time series or alerts when communicating with
+    # external systems (federation, remote storage, Alertmanager).
+    external_labels:
+      monitor: 'example'
+
+  # Alertmanager configuration
+  alerting:
+    alertmanagers:
+      - static_configs:
+          - targets: ['localhost:9093']
+
+  # Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+  rule_files:
+    # - "first_rules.yml"
+    # - "second_rules.yml"
+
+  # A scrape configuration containing exactly one endpoint to scrape:
+  # Here it's Prometheus itself.
+  scrape_configs:
+    # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+    - job_name: 'prometheus'
+
+      # Override the global default and scrape targets from this job every 5 seconds.
+      scrape_interval: 5s
+      scrape_timeout: 5s
+
+      # metrics_path defaults to '/metrics'
+      # scheme defaults to 'http'.
+
+      static_configs:
+        - targets: ['localhost:9090']
+
+    - job_name: node
+      # If prometheus-node-exporter is installed, grab stats about the local
+      # machine by default.
+      static_configs:
+        - targets: ['localhost:9100']
diff --git a/roles/prometheus/handlers/main.yml b/roles/prometheus/handlers/main.yml
new file mode 100644
index 0000000..b19a054
--- /dev/null
+++ b/roles/prometheus/handlers/main.yml
@@ -0,0 +1,14 @@
+---
+- name: reload the prometheus service
+  service:
+    name: prometheus
+    state: reloaded
+  tags:
+    - role::prometheus
+
+- name: restart the prometheus service
+  service:
+    name: prometheus
+    state: restarted
+  tags:
+    - role::prometheus
diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml
new file mode 100644
index 0000000..08aff38
--- /dev/null
+++ b/roles/prometheus/tasks/main.yml
@@ -0,0 +1,33 @@
+---
+- name: install prometheus
+  package:
+    name: prometheus
+    state: present
+  tags:
+    - role::prometheus
+
+- name: configure prometheus command line options
+  lineinfile:
+    path: /etc/default/prometheus
+    regexp: ^ARGS.*
+    line: ARGS="{{ prometheus_cmdline_options }}"
+  tags:
+    - role::prometheus
+  when:
+    - prometheus_cmdline_options is defined
+  notify:
+    - restart the prometheus service
+
+- name: configure prometheus
+  copy:
+    content: |
+      # Ansible managed
+      {{ prometheus_configuration | to_nice_yaml }}
+    dest: /etc/prometheus/prometheus.yml
+    owner: prometheus
+    group: prometheus
+    mode: 0400
+  tags:
+    - role::prometheus
+  notify:
+    - reload the prometheus service