Feed Dovecot maildir stats into Prometheus

Already deployed on lovelace. I was going to leave out script deployment
and just write it inline, but YAML folding of long lines (and
unreadability of 200 column-wide AWk scripts, to be honest) made it a
bit bad.

The e-mail for DevOps cron failure reports is updated to include `+cron`
to allow for client-side filtering, if necessary.

To test: `ssh -L localhost:9090:localhost:9090 lovelace.box.pydis.wtf`,
then check out the `dovecot_` variables in the UI.

We might want to further check out Dovecot's built-in statistics
support, see https://doc.dovecot.org/2.3/configuration_manual/stats/.

7 files changed, 95 insertions, 1 deletions

diff --git a/ansible/playbook.yml b/ansible/playbook.yml
index 728f930..2ccdd9b 100644
--- a/ansible/playbook.yml
+++ b/ansible/playbook.yml
@@ -23,6 +23,7 @@
     - opendmarc
     - sasl
     - dovecot
+    - dovecot-monitoring
     - spamassassin
     - postfix
     - prometheus-postfix-exporter
diff --git a/ansible/roles/dovecot-monitoring/defaults/main.yml b/ansible/roles/dovecot-monitoring/defaults/main.yml
new file mode 100644
index 0000000..28e0c33
--- /dev/null
+++ b/ansible/roles/dovecot-monitoring/defaults/main.yml
@@ -0,0 +1,3 @@
+---
+dovecot_monitoring_cron_filename: ansible_dovecot_prometheus_textfile_exporter
+dovecot_monitoring_scripts_directory: /opt/pydis/dovecot-monitoring
diff --git a/ansible/roles/dovecot-monitoring/meta/main.yml b/ansible/roles/dovecot-monitoring/meta/main.yml
new file mode 100644
index 0000000..8d8eb8c
--- /dev/null
+++ b/ansible/roles/dovecot-monitoring/meta/main.yml
@@ -0,0 +1,4 @@
+---
+dependencies:
+  - dovecot
+  - prometheus-node-exporter
diff --git a/ansible/roles/dovecot-monitoring/tasks/main.yml b/ansible/roles/dovecot-monitoring/tasks/main.yml
new file mode 100644
index 0000000..91f167a
--- /dev/null
+++ b/ansible/roles/dovecot-monitoring/tasks/main.yml
@@ -0,0 +1,52 @@
+---
+- name: Create dovecot monitoring directory
+  ansible.builtin.file:
+    path: "{{ dovecot_monitoring_scripts_directory }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  tags:
+    - role::dovecot-monitoring
+
+- name: Create dovecot monitoring scripts
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "{{ dovecot_monitoring_scripts_directory }}/{{ item }}"
+    owner: root
+    group: root
+    mode: "0544"
+  tags:
+    - role::dovecot-monitoring
+  loop:
+    - maildir-mails.sh
+    - maildir-sizes.sh
+
+- name: Create Maildir size monitoring cronjobs
+  ansible.builtin.cron:
+    name: "{{ item.name }}"
+    minute: "*/20"
+    hour: "*"
+    job: "{{ item.job }}"
+    cron_file: "{{ dovecot_monitoring_cron_filename }}"
+    user: root
+  tags:
+    - role::dovecot-monitoring
+  register: dovecot_monitoring_cron_file
+  loop_control:
+    label: "{{ item.name }}"
+  loop:
+    - name: Dovecot maildir size Prometheus exporter
+      job: "{{ dovecot_monitoring_scripts_directory }}/maildir-sizes.sh"
+    - name: Dovecot maildir mail count Prometheus exporter
+      job: "{{ dovecot_monitoring_scripts_directory }}/maildir-mails.sh"
+
+- name: Report to DevOps when Maildir size exporter fails
+  ansible.builtin.cron:
+    name: MAILTO
+    env: true
+    job: [email protected]
+    cron_file: "{{ dovecot_monitoring_cron_filename }}"
+    user: vmail
+  tags:
+    - role::dovecot-monitoring
diff --git a/ansible/roles/dovecot-monitoring/templates/maildir-mails.sh.j2 b/ansible/roles/dovecot-monitoring/templates/maildir-mails.sh.j2
new file mode 100644
index 0000000..37e8d17
--- /dev/null
+++ b/ansible/roles/dovecot-monitoring/templates/maildir-mails.sh.j2
@@ -0,0 +1,19 @@
+#!/bin/sh
+# Ansible managed
+
+cd /var/vmail && \
+  find . \
+  | awk -F / '
+    # Maildir e-mails have the hostname contained in them
+    $0 ~ "{{ ansible_fqdn }}" {
+      total[$2] += 1
+    }
+    END {
+      print "# HELP dovecot_maildir_mail_count Count of e-mails by user"
+      print "# TYPE dovecot_maildir_mail_count gauge"
+      for (user in total) {
+        print "dovecot_maildir_mail_count{user=\"" user "\"} " total[user]
+      }
+    }
+  ' \
+  | sponge > /var/lib/prometheus/node-exporter/dovecot-maildir-mails.prom
diff --git a/ansible/roles/dovecot-monitoring/templates/maildir-sizes.sh.j2 b/ansible/roles/dovecot-monitoring/templates/maildir-sizes.sh.j2
new file mode 100644
index 0000000..75086d0
--- /dev/null
+++ b/ansible/roles/dovecot-monitoring/templates/maildir-sizes.sh.j2
@@ -0,0 +1,15 @@
+#!/bin/sh
+# Ansible managed
+
+cd /var/vmail && \
+  du --bytes --summarize -- * \
+  | awk '
+    BEGIN {
+      print "# HELP dovecot_maildir_size_bytes Maildir size of e-mail users"
+      print "# TYPE dovecot_maildir_size_bytes gauge"
+    }
+    {
+      print "dovecot_maildir_size_bytes{user=\"" $2 "\"} " $1
+    }
+  ' \
+  | sponge > /var/lib/prometheus/node-exporter/dovecot-maildir-sizes.prom
diff --git a/ansible/roles/git-mirrors/vars/main.yml b/ansible/roles/git-mirrors/vars/main.yml
index e063d2e..da1f168 100644
--- a/ansible/roles/git-mirrors/vars/main.yml
+++ b/ansible/roles/git-mirrors/vars/main.yml
@@ -1,7 +1,7 @@
 ---
 git_mirrors_base_dir: "/srv/git-mirrors"
 git_mirrors_user: "git-mirrors"
-git_mirrors_error_email: "[email protected]"
+git_mirrors_error_email: "[email protected]"
 
 git_mirrors_cgit_logo: "https://raw.githubusercontent.com/python-discord/ops-site/main/src/images/icon.png"
 git_mirrors_cgit_title: PyDis DevOps Git Server