Merge branch 'backup'

.gitlab-ci.yml

 ---
 include:
-  - project: shore/ci-templates
+  - project: shore/ci-stuff
     file: templates/pre-commit.yml
-  - project: shore/ci-templates
+  - project: shore/ci-stuff
     file: templates/docker.yml
-
-stages:
-  - test
-  - build
-  - deploy
+  - project: shore/ci-stuff
+    file: templates/notify.yml
 
 build:
   extends: .compose-build
   tags: &tags [ns4.shore.co.il]
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
 
 pull:
   extends: .compose-pull
   tags: *tags
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
 
 run:
+  rules:
+    - if: $CI_PIPELINE_SOURCE != "schedule"
+      when: manual
   extends: .compose-run
   tags: *tags
+
+
+backup:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+  stage: deploy
+  tags: [host01.shore.co.il]
+  image: docker.io/library/docker:20.10
+  before_script:
+    - >-
+      docker build
+      --tag registry.shore.co.il/registry-backup
+      --pull
+      backup
+  script:
+    - >-
+      docker run
+      --volume /var/backups/registry:/var/backups/registry
+      --user nobody
+      --rm
+      registry.shore.co.il/registry-backup
+      backup registry.shore.co.il /var/backups/registry
+  retry:
+    max: 2
+  timeout: 3h

.pre-commit-config.yaml

@@ -54,3 +54,53 @@ repos:
     rev: v2.7.0
     hooks:
       - id: hadolint
+
+  - repo: https://github.com/ambv/black
+    rev: 21.9b0
+    hooks:
+      - id: black
+        args:
+          - |
+              --line-length=79
+
+  - repo: https://github.com/PyCQA/prospector
+    rev: 1.5.1
+    hooks:
+      - id: prospector
+        args:
+          - |-
+            --max-line-length=79
+          - |-
+            --with-tool=pyroma
+          - |-
+            --with-tool=bandit
+          - |-
+            --without-tool=pep257
+          - |-
+            --doc-warnings
+          - |-
+            --test-warnings
+          - |-
+            --full-pep8
+          - |-
+            --strictness=high
+          - |-
+            --no-autodetect
+        additional_dependencies:
+          - bandit
+          - pyroma
+
+  - repo: https://gitlab.com/pycqa/flake8.git
+    rev: 3.9.2
+    hooks:
+      - id: flake8
+        args:
+          - |-
+            --doctests
+        additional_dependencies:
+          - flake8-bugbear
+
+  - repo: https://github.com/codespell-project/codespell.git
+    rev: v2.1.0
+    hooks:
+      - id: codespell

backup/.dockerignore

+*
+!backup
+!restore

backup/Dockerfile

+FROM docker.io/library/alpine:3.14
+# hadolint ignore=DL3018
+RUN echo 'https://dl-cdn.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories && \
+    echo 'https://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories && \
+    apk add --update --no-cache \
+        findutils \
+        skopeo \
+        reg \
+    ;
+COPY --chown=root:root backup /usr/local/bin/backup
+COPY --chown=root:root restore /usr/local/bin/restore

backup/backup

+#!/bin/sh
+set -eu
+
+usage() {
+    echo "$0: REGISTRY_DOMAIN BACKUP_DEST"
+}
+
+if [ "${1:-}" = -h ] || [ "${1:-}" = --help ]
+then
+    usage
+    exit 0
+fi
+
+if [ "$#" -ne 2 ]
+then
+    usage
+    exit 1
+fi
+
+command -v skopeo >/dev/null || { echo 'skopeo is missing.' >&2; exit 2; }
+
+registry="$1"
+dest="$2"
+
+mkdir -p "$dest"
+
+reg ls "$registry" | \
+   sed 's/,//g' | \
+   awk -v "registry=$registry" -v "dest=$dest" '
+BEGIN {
+    exitcode = 0
+}
+NR>2 {
+    system("mkdir -p " dest "/" $1)
+    for (i=2; i<=NF; i++) {
+        image_url = registry "/" $1 ":" $(i)
+        image_file = dest "/" $1 "/" $(i) ".tar"
+        printf "Saving %s to %s.\n", image_url, image_file
+        system("rm " image_file)
+        if (system("skopeo copy docker://" image_url " docker-archive://" image_file) == 0)
+            printf "Backup of %s was successful.\n", image_url
+        else {
+            exitcode = 1
+            printf "Backup of %s failed, continuing with other images.\n", image_url
+        }
+    }
+}
+END {
+    if ( exitcode == 1) print "Backup failed for some images."
+    exit exitcode
+}
+'

backup/restore

+#!/bin/sh
+set -eu
+
+usage() {
+    echo "$0: BACKUP_SOURCE REGISTRY_DOMAIN"
+}
+
+if [ "${1:-}" = -h ] || [ "${1:-}" = --help ]
+then
+    usage
+    exit 0
+fi
+
+if [ "$#" -ne 2 ]
+then
+    usage
+    exit 1
+fi
+
+command -v skopeo >/dev/null || { echo 'skopeo is missing.' >&2; exit 2; }
+
+src="$1"
+registry="$2"
+
+# There's an assumption here that filenames don't have spaces (or other such
+# characters) as the image format prohibits that and I don't want to deal with
+# such issues right now.
+
+images="$(find "$src" -maxdepth 1 -mindepth 1 -type d -printf '%f\n')"
+if [ -z "$images" ]
+then
+    echo 'No images found,' >&2
+    exit 3
+fi
+
+returncode=0
+for image in $images
+do
+    tags="$(find "$src/$image" -maxdepth 1 -mindepth 1 -type f -name '*.tar' -printf '%f\n' | sed 's/\.tar$//g')"
+    if [ -z "$tags" ]
+    then
+        echo "No tags found for image $image, skipping." >&2
+        continue
+    fi
+    for tag in $tags
+    do
+        echo "Restoring $image:$tag" >&2
+        if skopeo copy "docker-archive://$src/$image/$tag.tar" "docker://$registry/$image:$tag"
+        then
+            echo "Restore finished successfully." >&2
+        else
+            echo "Restore failed, continuing with other image." >&2
+            returncode=1
+        fi
+    done
+done
+
+if [ "$returncode" -gt 0 ]
+then
+    echo 'Restoration failed for some images.'
+fi
+exit "$returncode"