Skip to content
Snippets Groups Projects
Commit 78d9614b authored by nimrod's avatar nimrod
Browse files

Maintenance tasks of Debian servers.

- mnpw is now working (enough) so let's put it to use for all of the
  various Nagios-style checks.
- Add the monitoring plugins from monitoring-plugins.org/ for check_apt.
  Now I should get better alerts for pending updates. Regular updates
shuold be just a warning but security updates should be a critical
alert. Let's put it in production and see.
- Now with check_apt handling APT updates, write a small shell script
  for a Nagios-style check for firmware updates.
- And the last part of the updates, update flatpaks silently.
- Refactor of the general maintenance and btrfs-specific tasks.
- Tag the include_tasks tasks with always, so those tasks are always
  included so their tags are also evaluated.
- Remove the apt-transport-https package installation, it's a dummy
  package for apt since Debian Buster and Ubuntu Bionic.
parent f9903a11
No related branches found
No related tags found
No related merge requests found
Showing
with 202 additions and 103 deletions
#!/bin/sh
set -eu
apt-get update
apt-get dist-upgrade --download-only --yes
apt-get autoclean
#!/bin/sh
set -eu
apt-get update
apt-get dist-upgrade --download-only --yes
apt-get autoclean
if command -v flatpak >/dev/null 2>&1
then
flatpak --system update --appstream
flatpak --system update --assumeyes
flatpak --system uninstall --unused --assumeyes
fi
fwupdmgr --assume-yes get-updates
if [ "$(apt list --upgradable 2>/dev/null | wc -l)" -gt '1' ] || \
[ -n "$(fwupdmgr --assume-yes get-updates)" ]
then
curl "https://notify.shore.co.il/send?message=$(hostname --short)%20has%20pending%20updates."
fi
#!/bin/sh
set -eu
if ! which flatpak >/dev/null
then
echo 'Flatpak not installed, exiting.'
exit
fi
flatpak --system update --appstream
flatpak --system update --assumeyes
flatpak --system uninstall --unused --assumeyes
#!/bin/sh
set -eu
# First to download an updated list silently.
fwupdmgr --assume-yes get-updates 1>/dev/null
# Now to check if there are pending updates.
if [ "$(fwupdmgr --assume-yes get-updates | wc -l)" -gt 1 ]
then
echo 'Available firmware updates.'
exit 2
else
echo 'Firmware is up to date.'
exit 0
fi
# vim: filetype=systemd
[Unit]
Description=Check if a restart is needed
ConditionACPower=true
After=network-online.target
[Service]
Type=oneshot
ExecStart=/bin/sh -c 'needrestart -p || curl "https://notify.shore.co.il/send?message=$(hostname --short)%%20needs%%20to%%20be%%20restarted."'
---
- name: Install btrfs packages
apt:
cache_valid_time: 3600
name:
- btrfs-progs
- monitoring-plugins-btrfs
state: present
- name: Copy btrfs backup script
copy:
dest: /usr/local/sbin/btrfs-backup
......@@ -7,34 +15,37 @@
owner: root
src: btrfs-backup
- name: Create btrfs scrub services
# yamllint disable rule:line-length
loop: &btrfs_devices |-
{{ ansible_facts.mounts|selectattr("fstype", "equalto", "btrfs")|map(attribute="device")|unique }}
# yamllint enable rule:line-length
- name: Install btrfs scrub services and timers
loop: &loop
- service
- timer
ansible.builtin.template:
dest: /etc/systemd/system/scrub{{ name }}.service
dest: /etc/systemd/system/btrfs_scrub{{ name }}.{{ item }}
mode: 0o0644
src: scrub.service.j2
vars: &btrfs_vars
device: '{{ item }}'
name: '{{ item|replace("/", "_") }}'
src: btrfs_scrub.{{ item }}.j2
vars: &vars
name: '{{ device|replace("/", "_") }}'
tags: [scrub]
- name: Enable btrfs scrub timers
ansible.builtin.systemd:
enabled: true
name: btrfs_scrub{{ name }}.timer
state: started
vars: *vars
tags: [scrub]
- name: Create btrfs scrub timers
loop: *btrfs_devices
- name: Install btrfs check services and timers
loop: *loop
ansible.builtin.template:
dest: /etc/systemd/system/scrub{{ name }}.timer
dest: /etc/systemd/system/btrfs_check{{ name }}.{{ item }}
mode: 0o0644
src: scrub.timer.j2
vars: *btrfs_vars
tags: [scrub]
src: btrfs_check.{{ item }}.j2
vars: *vars
- name: Enable the btrfs scrub timers
loop: *btrfs_devices
- name: Enable the btrfs check timers
ansible.builtin.systemd:
enabled: true
name: scrub{{ name }}.timer
state: started
vars: *btrfs_vars
tags: [scrub]
vars: *vars
......@@ -32,16 +32,13 @@
cache_valid_time: 3600
name:
- amd64-microcode
- btrfs-progs
- cryptsetup
- curl
- docker.io
- fwupd
- git
- intel-microcode
- libpam-mount-bin
- make
- needrestart
- python3-cryptography
- rsync
- tmux
......@@ -66,7 +63,7 @@
value: "640000"
- name: Configure the SSH daemon
include_tasks:
ansible.builtin.include_tasks:
file: '{{ playbook_dir }}/tasks/sshd_config.yaml'
tags:
- always
......@@ -98,58 +95,38 @@
regexp: '^{{ item }}:'
state: present
- name: Copy update script
ansible.builtin.copy:
dest: /usr/local/sbin/daily
mode: 0o0755
src: daily
- name: Copy update service and timer
loop:
- update.service
- update.timer
ansible.builtin.copy:
dest: /etc/systemd/system
mode: 0o0644
src: '{{ item }}'
notify:
- Systemd daemon reload
- name: Enable the update timer
ansible.builtin.systemd:
enabled: true
name: update.timer
state: started
- name: Copy the need restart service and timer
loop:
- needrestart.service
- needrestart.timer
ansible.builtin.copy:
dest: /etc/systemd/system
mode: 0o0644
src: '{{ item }}'
notify:
- Systemd daemon reload
- name: Enable the need restart timer
ansible.builtin.systemd:
enabled: true
name: needrestart.timer
state: started
- name: Include Docker tasks
ansible.builtin.include_tasks:
file: docker.yml
tags:
- always
- name: Include web tasks
ansible.builtin.include_tasks:
file: web.yml
tags:
- always
- name: Include maintenance tasks
ansible.builtin.include_tasks:
file: maintenance.yml
tags:
- always
- name: Include btrfs tasks
# yamllint disable rule:line-length
loop: |-
{{ ansible_facts.mounts|selectattr("fstype", "equalto", "btrfs")|map(attribute="device")|unique }}
# yamllint enable rule:line-length
loop_control:
loop_var: device
ansible.builtin.include_tasks:
file: btrfs.yml
tags:
- always
- name: Include Toolbox tasks
ansible.builtin.import_tasks:
file: toolbox.yml
tags:
- always
---
- name: Install maintenance packages
ansible.builtin.apt:
cache_valid_time: 3600
install_recommends: false
name:
- fwupd
- monitoring-plugins-basic
- needrestart
state: present
- name: Download My Nagios Plugin Wrapper
ansible.builtin.get_url:
dest: /usr/local/bin/mnpw
mode: 0o0755
# yamllint disable-line rule:line-length
url: https://git.shore.co.il/api/v4/projects/167/packages/generic/mnpw/v0.1.2/mnpw
- name: Create a directory for periodic scripts and Nagios plugins
ansible.builtin.file:
mode: 0o0755
path: /usr/local/share/shore-maintenance
state: directory
- name: Copy periodic scripts and Nagios plugins
loop:
- apt-download-updates
- flatpak-update
- fw-update
ansible.builtin.copy:
dest: /usr/local/share/shore-maintenance/
group: root
mode: 0o0755
owner: root
src: '{{ item }}'
- name: Install Systemd services and timers for maintenance tasks
loop: |-
{{ maintenance_tasks|product(["service", "timer"]) }}
vars:
command: '{{ item[0]["command"] }}'
description: '{{ item[0]["description"] }}'
frequency: '{{ item[0]["frequency"] }}'
name: '{{ item[0]["name"] }}'
unit: '{{ item[1] }}'
ansible.builtin.template:
dest: /etc/systemd/system/{{ name }}.{{ unit }}
mode: 0o0644
src: maintenance-task.{{ unit }}.j2
notify:
- Systemd daemon reload
- name: Enable the Systemd timers for maintenance tasks
loop: '{{ maintenance_tasks }}'
ansible.builtin.systemd:
enabled: true
name: '{{ name }}.timer'
state: started
vars:
name: '{{ item["name"] }}'
notify:
- Systemd daemon reload
......@@ -28,14 +28,6 @@
that:
- toolbox_podman_repo_base_url is defined
- name: APT install prerequisites
apt:
cache_valid_time: 3600
name:
- apt-transport-https
state: present
update_cache: true
- name: Add the Podman repo key
apt_key:
state: present
......
# vim: filetype=systemd
[Unit]
Description=Check btrfs device {{ device }}
After=local-fs.target
After=network-online.target
[Service]
Type=exec
ExecStart=mnpw --warn -- /usr/lib/nagios-btrfs/plugins/check_btrfs --mountpoint {{ device }}
# vim: filetype=systemd
[Unit]
Description=Background system updates
Description=Check btrfs device {{ device }}
[Timer]
OnCalendar=daily
......
# vim: filetype=systemd
[Unit]
Description=Background system updates
Description={{ description|default(name) }}
ConditionACPower=true
After=local-fs.target
After=network-online.target
[Service]
Type=oneshot
ExecStart=daily
Type=exec
ExecStart={{ command }}
# vim: filetype=systemd
[Unit]
Description=Check if a restart is needed
Description={{ description|default(name) }}
[Timer]
OnCalendar=daily
OnCalendar={{ frequency|default("daily") }}
[Install]
WantedBy=multi-user.target
---
maintenance_directory: /usr/local/share/shore-maintenance
maintenance_tasks:
- name: apt-check
description: Check for pending APT updates
command: mnpw --errors --warn -- /usr/lib/nagios/plugins/check_apt --list
frequency: daily
- name: apt-download-updates
description: Download pending APT updates for faster updates
command: '{{ maintenance_directory }}/apt-download-updates'
frequency: daily
- name: flatpak-update
description: Update flatpaks
command: '{{ maintenance_directory }}/flatpak-update'
frequency: daily
- name: fw-update
description: Check for available firmware updates
command: mnpw --errors -- {{ maintenance_directory }}/fw-update
frequency: daily
- name: needrestart
description: Check if a restart is needed
command: mnpw --errors --warn -- needrestart -p
frequency: daily
# Hera and Odin are ElementaryOS releases based on Ubuntu Bionic and Focal
# respectively.
toolbox_podman_repo_base_url_mapping:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment