diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 70cfcadb05eed8c528f9e0984bc1eb2f572eb1f7..e8d010bd2e12dff9c16febab4aa9725052bfd2e3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,23 +1,52 @@
---
include:
- - project: shore/ci-templates
+ - project: shore/ci-stuff
file: templates/pre-commit.yml
- - project: shore/ci-templates
+ - project: shore/ci-stuff
file: templates/docker.yml
-
-stages:
- - test
- - build
- - deploy
+ - project: shore/ci-stuff
+ file: templates/notify.yml
build:
extends: .compose-build
tags: &tags [ns4.shore.co.il]
+ rules:
+ - if: $CI_PIPELINE_SOURCE != "schedule"
pull:
extends: .compose-pull
tags: *tags
+ rules:
+ - if: $CI_PIPELINE_SOURCE != "schedule"
run:
+ rules:
+ - if: $CI_PIPELINE_SOURCE != "schedule"
+ when: manual
extends: .compose-run
tags: *tags
+
+
+backup:
+ rules:
+ - if: $CI_PIPELINE_SOURCE == "schedule"
+ stage: deploy
+ tags: [host01.shore.co.il]
+ image: docker.io/library/docker:20.10
+ before_script:
+ - >-
+ docker build
+ --tag registry.shore.co.il/registry-backup
+ --pull
+ backup
+ script:
+ - >-
+ docker run
+ --volume /var/backups/registry:/var/backups/registry
+ --user nobody
+ --rm
+ registry.shore.co.il/registry-backup
+ backup registry.shore.co.il /var/backups/registry
+ retry:
+ max: 2
+ timeout: 3h
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 1b8468929bf63482ead46185326d9a0a8ca927c4..5ae94b53dadf1ffd48f779f4e46da01d98c3ed56 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -54,3 +54,53 @@ repos:
rev: v2.7.0
hooks:
- id: hadolint
+
+ - repo: https://github.com/ambv/black
+ rev: 21.9b0
+ hooks:
+ - id: black
+ args:
+ - |
+ --line-length=79
+
+ - repo: https://github.com/PyCQA/prospector
+ rev: 1.5.1
+ hooks:
+ - id: prospector
+ args:
+ - |-
+ --max-line-length=79
+ - |-
+ --with-tool=pyroma
+ - |-
+ --with-tool=bandit
+ - |-
+ --without-tool=pep257
+ - |-
+ --doc-warnings
+ - |-
+ --test-warnings
+ - |-
+ --full-pep8
+ - |-
+ --strictness=high
+ - |-
+ --no-autodetect
+ additional_dependencies:
+ - bandit
+ - pyroma
+
+ - repo: https://gitlab.com/pycqa/flake8.git
+ rev: 3.9.2
+ hooks:
+ - id: flake8
+ args:
+ - |-
+ --doctests
+ additional_dependencies:
+ - flake8-bugbear
+
+ - repo: https://github.com/codespell-project/codespell.git
+ rev: v2.1.0
+ hooks:
+ - id: codespell
diff --git a/backup/.dockerignore b/backup/.dockerignore
new file mode 100644
index 0000000000000000000000000000000000000000..5664021a5516007889500c3fb8d99da128793e9a
--- /dev/null
+++ b/backup/.dockerignore
@@ -0,0 +1,3 @@
+*
+!backup
+!restore
diff --git a/backup/Dockerfile b/backup/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..274cba84e3eb9f1276bc754ff3bb2af9f3f763b3
--- /dev/null
+++ b/backup/Dockerfile
@@ -0,0 +1,11 @@
+FROM docker.io/library/alpine:3.14
+# hadolint ignore=DL3018
+RUN echo 'https://dl-cdn.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories && \
+ echo 'https://dl-cdn.alpinelinux.org/alpine/edge/community' >> /etc/apk/repositories && \
+ apk add --update --no-cache \
+ findutils \
+ skopeo \
+ reg \
+ ;
+COPY --chown=root:root backup /usr/local/bin/backup
+COPY --chown=root:root restore /usr/local/bin/restore
diff --git a/backup/backup b/backup/backup
new file mode 100755
index 0000000000000000000000000000000000000000..fe42de0fadd0b25e7a8952124f3626c0c687c8c5
--- /dev/null
+++ b/backup/backup
@@ -0,0 +1,52 @@
+#!/bin/sh
+set -eu
+
+usage() {
+ echo "$0: REGISTRY_DOMAIN BACKUP_DEST"
+}
+
+if [ "${1:-}" = -h ] || [ "${1:-}" = --help ]
+then
+ usage
+ exit 0
+fi
+
+if [ "$#" -ne 2 ]
+then
+ usage
+ exit 1
+fi
+
+command -v skopeo >/dev/null || { echo 'skopeo is missing.' >&2; exit 2; }
+
+registry="$1"
+dest="$2"
+
+mkdir -p "$dest"
+
+reg ls "$registry" | \
+ sed 's/,//g' | \
+ awk -v "registry=$registry" -v "dest=$dest" '
+BEGIN {
+ exitcode = 0
+}
+NR>2 {
+ system("mkdir -p " dest "/" $1)
+ for (i=2; i<=NF; i++) {
+ image_url = registry "/" $1 ":" $(i)
+ image_file = dest "/" $1 "/" $(i) ".tar"
+ printf "Saving %s to %s.\n", image_url, image_file
+ system("rm " image_file)
+ if (system("skopeo copy docker://" image_url " docker-archive://" image_file) == 0)
+ printf "Backup of %s was successful.\n", image_url
+ else {
+ exitcode = 1
+ printf "Backup of %s failed, continuing with other images.\n", image_url
+ }
+ }
+}
+END {
+ if ( exitcode == 1) print "Backup failed for some images."
+ exit exitcode
+}
+'
diff --git a/backup/restore b/backup/restore
new file mode 100755
index 0000000000000000000000000000000000000000..6dc6cf0daa29bd664fca6b8eccc5b62be856eb61
--- /dev/null
+++ b/backup/restore
@@ -0,0 +1,62 @@
+#!/bin/sh
+set -eu
+
+usage() {
+ echo "$0: BACKUP_SOURCE REGISTRY_DOMAIN"
+}
+
+if [ "${1:-}" = -h ] || [ "${1:-}" = --help ]
+then
+ usage
+ exit 0
+fi
+
+if [ "$#" -ne 2 ]
+then
+ usage
+ exit 1
+fi
+
+command -v skopeo >/dev/null || { echo 'skopeo is missing.' >&2; exit 2; }
+
+src="$1"
+registry="$2"
+
+# There's an assumption here that filenames don't have spaces (or other such
+# characters) as the image format prohibits that and I don't want to deal with
+# such issues right now.
+
+images="$(find "$src" -maxdepth 1 -mindepth 1 -type d -printf '%f\n')"
+if [ -z "$images" ]
+then
+ echo 'No images found,' >&2
+ exit 3
+fi
+
+returncode=0
+for image in $images
+do
+ tags="$(find "$src/$image" -maxdepth 1 -mindepth 1 -type f -name '*.tar' -printf '%f\n' | sed 's/\.tar$//g')"
+ if [ -z "$tags" ]
+ then
+ echo "No tags found for image $image, skipping." >&2
+ continue
+ fi
+ for tag in $tags
+ do
+ echo "Restoring $image:$tag" >&2
+ if skopeo copy "docker-archive://$src/$image/$tag.tar" "docker://$registry/$image:$tag"
+ then
+ echo "Restore finished successfully." >&2
+ else
+ echo "Restore failed, continuing with other image." >&2
+ returncode=1
+ fi
+ done
+done
+
+if [ "$returncode" -gt 0 ]
+then
+ echo 'Restoration failed for some images.'
+fi
+exit "$returncode"