Ansible: Add failure notification to Debian servers.

Rather cool. I've seen it before and had it on my todo list.

Ansible/roles/debian_server/files/dvb.service

@@ -3,6 +3,7 @@
 Description=Docker volume backup (dvb)
 ConditionACPower=true
 After=local-fs.target
+OnFailure=failure-notify@%n.service
 
 [Service]
 Type=exec

Ansible/roles/debian_server/files/failure-handler@.service

+[Unit]
+Description=Failure handler for %i
+
+[Service]
+Type=oneshot
+ExecStart=curl "https://notify.shore.co.il/send?message=Service%20$ifailed%20on%20%H."

Ansible/roles/debian_server/files/failure-notify@.service

+[Unit]
+Description=Send failure notice for %i
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=curl https://notify.shore.co.il/send?message=Service%%20%i%%20on%%20%q%%20failed.
+DynamicUser=yes
+PrivateDevices=yes
+ProtectSystem=strict
+ProtectHome=yes
+PrivateTmp=yes
+PrivateIPC=yes
+ProtectProc=invisible
+PrivateUsers=yes

Ansible/roles/debian_server/tasks/main.yml

@@ -148,3 +148,42 @@
         - toolbx
   tags:
     - always
+
+- name: Add the failure notify service
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/
+    group: root
+    mode: preserve
+    owner: root
+    src: failure-notify@.service
+  tags: [notify]
+  notify:
+    - Systemd daemon reload
+
+- name: Create services drop-in directories for failure notifications
+  loop: &notify_failure_units
+    - containerd.service
+    - cron.service
+    - docker.service
+    - rsyslog.service
+    - ssh.service
+    - systemd-networkd.service
+    - systemd-timesyncd.service
+    - ufw.service
+  ansible.builtin.file:
+    path: /etc/systemd/system/{{ item }}.d
+    mode: 0o0755
+    state: directory
+  tags: [notify]
+
+- name: Add failure notification to services
+  loop: *notify_failure_units
+  ansible.builtin.copy:
+    content: |
+      [Unit]
+      OnFailure=failure-notify@%n.service
+    dest: /etc/systemd/system/{{ item }}.d/failure-notify.conf
+    mode: 0o0644
+  tags: [notify]
+  notify:
+    - Systemd daemon reload

Ansible/roles/debian_server/tasks/maintenance.yml

@@ -60,3 +60,9 @@
     name: '{{ item["name"] }}'
   notify:
     - Systemd daemon reload
+
+- name: Create the directory for service overrides
+  ansible.builtin.file:
+    mode: 0o0755
+    path: /etc/systemd/system/service.d
+    state: directory

Ansible/roles/debian_server/templates/btrfs_check.service.j2

@@ -3,6 +3,7 @@
 Description=Check btrfs device {{ device }}
 After=local-fs.target
 After=network-online.target
+OnFailure=failure-notify@%n.service
 
 [Service]
 Type=exec

Ansible/roles/debian_server/templates/btrfs_scrub.service.j2

@@ -3,6 +3,7 @@
 Description=Scrub btrfs device {{ device }}
 ConditionACPower=true
 After=local-fs.target
+OnFailure=failure-notify@%n.service
 
 [Service]
 Type=exec

Ansible/roles/debian_server/templates/maintenance-task.service.j2

@@ -4,6 +4,7 @@ Description={{ description|default(name) }}
 ConditionACPower=true
 After=local-fs.target
 After=network-online.target
+OnFailure=failure-notify@%n.service
 
 [Service]
 Type=exec