all: Add new role ffh.restic.

For most of the nodes, this only backups /etc/. Some nodes specify
the restic_backup_paths variable in their host_vars and backup additional
paths.

Currently this is not rolled out yet, as @1997er has to create the
backup machine first. However, once he is finished, he can adjust the
remaining variables (marked with "TODO") in group_vars/all/backups.yml.
Then he can roll all playbooks with "-t restic" and things should work
out smoothly. Let's see...

Some discussion about this can be found in #118.

group_vars/all/backups.yml

@@ -0,0 +1,14 @@
+---
+# This is overwritten for some hosts in host_vars/
+restic_backup_paths:
+- /etc/
+restic_backup_server_ips:
+  - 2a01:4f8:141:1464::666
+# TODO: change this ssh_figerprint to the correct one...
+#       use "cat /etc/ssh/ssh_host_ecdsa_key.pub" on the backup machine to find it...
+restic_backup_server_ssh_fingerprint: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCymC74C0M/JV1sJLjxiYV10yEyJe99UF3x4u4eAAk35ZuaxCh6qjQ6Stjui/RhKgMVppgtp1HC/B/Ix10YMoDk=
+# TODO: give me a better name
+restic_backup_server_ansible_host: backups
+restic_backup_server_hostname: 2a01:4f8:141:1464::666
+restic_backup_server_port: 1337
+restic_version: 0.9.6

host_vars/ns1.yml

@@ -149,3 +149,7 @@ postfix_mynetworks:
 - "[{{ lookup('dig', 'monitor.ffh.zone./AAAA') or '::1' }}]/128"
 - "{{ lookup('dig', 'harvester.ffh.zone./A') or '127.0.0.1' }}/32"
 - "[{{ lookup('dig', 'harvester.ffh.zone./AAAA') or '::1' }}]/128"
+
+restic_backup_paths:
+- /etc/
+- /home/git # gitolite

host_vars/web.yml

@@ -99,3 +99,7 @@ git2cal:
   calendar_dir: /tmp/calendar
   ics_file: /var/www/web/api/meetings.ics
   json_file: /var/www/web/api/meetings.json
+
+restic_backup_paths:
+- /etc/
+- /var/www/ # wiki (and other probably not so important stuff)

playbooks/harvester.yml

@@ -19,3 +19,5 @@
     - { name: ffh.yanic, tags: yanic }
     - { name: ffh.nginx, tags: nginx }
     - { name: ffh.zabbix-agent, tags: zabbix-agent }
+    # Backups
+    - { name: ffh.restic, tags: restic }

playbooks/leintor.yml

@@ -15,3 +15,5 @@
     - { name: ffh.apinger, tags: apinger }
     - { name: ffh.routingnode, tags: routingnode }
     - { name: ffh.telegraf, tags: telegraf }
+    # Backups
+    - { name: ffh.restic, tags: restic }

playbooks/monitor.yml

@@ -17,3 +17,5 @@
     - { name: ffh.zabbix-server, tags: zabbix-server }
     - { name: ffh.zabbix-frontend, tags: zabbix-frontend }
     - { role: ffh.nginx, tags: nginx }
+    # Backups
+    - { name: ffh.restic, tags: restic }

playbooks/ns1.yml

@@ -13,3 +13,5 @@
     - { name: ffh.postfix, tags: postfix }
     - { name: ffh.ntp, tags: ntp }
     - { name: ffh.zabbix-agent, tags: zabbix-agent }
+    # Backups
+    - { name: ffh.restic, tags: restic }

playbooks/supernodes.yml

@@ -49,6 +49,8 @@
     - { name: ffh.telegraf, tags: telegraf }
     - { name: ffh.routingnode, tags: routingnode }
     - { name: ffh.bpfcount, tags: bpfcount }
+    # Backups
+    - { name: ffh.restic, tags: restic }
   post_tasks:
     - name: etckeeper post-commit
       shell: 'etckeeper commit "post-commit changes after Ansible run ({{local_username.stdout}}, tags: {{ansible_run_tags | join(",")}})" || echo "nothing to commit"'

playbooks/web.yml

@@ -18,3 +18,5 @@
     # - { role: ffh.taskserver, tags: taskserver }
     - { name: ffh.zabbix-agent, tags: zabbix-agent }
     - { name: ffh.burginfo, tags: burginfo }
+    # Backups
+    - { name: ffh.restic, tags: restic }

roles/ffh.restic/tasks/main.yml

@@ -0,0 +1,123 @@
+---
+
+- name: add server to known_hosts
+  known_hosts:
+    path: /etc/ssh/ssh_known_hosts
+    name: "{{ item }}"
+    key: "{{ item }} {{ restic_backup_server_ssh_fingerprint }}"
+  loop: "{{ restic_backup_server_ips }}"
+
+- name: Check if passwords/{{ansible_hostname}} exists
+  local_action: stat path=passwords/{{ansible_hostname}}
+  register: passwordfile
+
+- name: Generate and store password
+  local_action: shell pwgen -y -1 20 | ansible-vault encrypt - > passwords/{{ansible_hostname}}
+  when: not passwordfile.stat.exists
+
+- name: Get password
+  local_action: shell ansible-vault view passwords/{{ansible_hostname}} | base64
+  changed_when: False
+  register: password
+
+- name: Create user on {{ restic_backup_server_ansible_host }}
+  user:
+    name: "{{ ansible_hostname }}"
+    groups: ["sftponly"]
+    force: no
+  delegate_to: "{{ restic_backup_server_ansible_host }}"
+
+- name: Create ssh-key
+  user:
+    name: "{{ansible_user}}"
+    generate_ssh_key: yes
+    ssh_key_type: rsa
+    ssh_key_bits: 4096
+    force: no
+
+- name: Obtain ssh-key
+  command: cat ~/.ssh/id_rsa.pub
+  changed_when: False
+  register: ssh_key
+
+- name: Create ssh folder on backups for user
+  file: path=/home/{{ansible_hostname}}/.ssh state=directory
+  delegate_to: "{{ restic_backup_server_ansible_host }}"
+
+- name: Add ssh-key
+  copy:
+    content: "{{ ssh_key.stdout }}"
+    dest: /home/{{ansible_hostname}}/.ssh/authorized_keys
+  delegate_to: "{{ restic_backup_server_ansible_host }}"
+
+- name: Check if restic data path exists
+  stat: path=/home/{{ansible_hostname}}/restic
+  register: restic_datapath
+  delegate_to: "{{ restic_backup_server_ansible_host }}"
+
+- name: Create folders and set permissions
+  shell: "cd /home/{{ansible_hostname}} && mkdir -p restic && chown root:root . && chown {{ansible_hostname}}:{{ansible_hostname}} restic"
+  delegate_to: "{{ restic_backup_server_ansible_host }}"
+  when: not restic_datapath.stat.exists
+
+- name: Check if restic is installed
+  stat: path=/usr/local/bin/restic
+  register: resticpath
+
+- name: Ensure restic is installed
+  block:
+  - get_url:
+      url: "https://github.com/restic/restic/releases/download/v{{restic_version}}/restic_{{restic_version}}_linux_amd64.bz2"
+      dest: /usr/local/bin/restic.bz2
+  - shell: "bzip2 -d /usr/local/bin/restic.bz2 && chmod +x /usr/local/bin/restic"
+  when: not resticpath.stat.exists
+
+- name: Install backups ssh host host
+  blockinfile:
+    path: /etc/ssh/ssh_config
+    block: |
+      Host backups
+          HostName {{ restic_backup_server_hostname }}
+          User {{ ansible_hostname }}
+          Port {{ restic_backup_server_port }}
+
+- name: Install restic config
+  template:
+    dest: /etc/restic.sh
+    src: backup.sh.j2
+    mode: 755
+
+- name: Install restic conf
+  template:
+    dest: /etc/restic-backup.conf
+    src: "restic-backup.conf.j2"
+
+- name: Init repo
+  command: /etc/restic.sh init
+  when: not restic_datapath.stat.exists
+
+- name: Install restic systemd units
+  template:
+    dest: /etc/systemd/system/{{ item }}
+    src: "{{ item }}"
+  with_items:
+    - restic-backup.service
+    - restic-backup.timer
+    - restic-prune.service
+    - restic-prune.timer
+  register: systemdunits
+
+- name: Enable systemd units
+  shell: >
+    set -e
+    systemctl daemon-reload
+    systemctl enable restic-backup
+    systemctl enable restic-prune
+    systemctl enable --now restic-backup.timer
+    systemctl enable --now restic-prune.timer
+    systemctl start restic-backup.timer # for legacy systemd where --now does not work
+    systemctl start restic-prune.timer # for legacy systemd where --now does not work
+  when: systemdunits.changed
+
+- name: Install fuse (for mounting)
+  apt: name=fuse state=present

roles/ffh.restic/templates/backup.sh.j2

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+export RESTIC_REPOSITORY=sftp:backups:/restic
+export RESTIC_PASSWORD="$(echo -n '{{password.stdout}}' | base64 -d)"
+
+/usr/local/bin/restic $@

roles/ffh.restic/templates/restic-backup.conf.j2

@@ -0,0 +1,6 @@
+BACKUP_PATHS="{{ restic_backup_paths | join(' ') }}"
+BACKUP_EXCLUDES=""
+RETENTION_DAYS=7
+RETENTION_WEEKS=4
+RETENTION_MONTHS=6
+RETENTION_YEARS=3

roles/ffh.restic/templates/restic-backup.service

@@ -0,0 +1,7 @@
+[Unit]
+Description=Restic backup service
+[Service]
+Type=oneshot
+ExecStart=/etc/restic.sh backup --verbose --one-file-system --tag systemd.timer $BACKUP_EXCLUDES $BACKUP_PATHS
+ExecStartPost=/etc/restic.sh forget --verbose --tag systemd.timer --group-by "paths,tags" --keep-daily $RETENTION_DAYS --keep-weekly $RETENTION_WEEKS --keep-monthly $RETENTION_MONTHS --keep-yearly $RETENTION_YEARS
+EnvironmentFile=/etc/restic-backup.conf

roles/ffh.restic/templates/restic-backup.timer

@@ -0,0 +1,6 @@
+[Unit]
+Description=Backup with restic daily
+[Timer]
+OnCalendar=daily
+[Install]
+WantedBy=timers.target

roles/ffh.restic/templates/restic-prune.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Restic backup service (data pruning)
+[Service]
+Type=oneshot
+ExecStart=/etc/restic.sh prune
+EnvironmentFile=/etc/restic-backup.conf

roles/ffh.restic/templates/restic-prune.timer

@@ -0,0 +1,6 @@
+[Unit]
+Description=Prune data from the restic repository monthly
+[Timer]
+OnCalendar=monthly
+[Install]
+WantedBy=timers.target