public
nobgit
read
NobMail
Based on mailcow: dockerized
Languages
Repository composition by tracked source files.
PHP
49%
JavaScript
35%
HTML
9%
CSS
4%
Shell
2%
Python
1%
Lua
0%
Perl
0%
Ruby
0%
SCSS
0%
Create file
Wiki Documentation
Clone
https://nobgit.com/orgs/nobgit/nobmail.git
ssh://[email protected]:2222/orgs/nobgit/nobmail.git
Commit
[Helper] Cold standby script (WIP, docs incoming)
78aafe9b
create_cold_standby.sh | 7 ++
helper-scripts/_cold-standby.sh | 251 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 258 insertions(+)
create mode 100644 create_cold_standby.sh
create mode 100755 helper-scripts/_cold-standby.sh
Diff
diff --git a/create_cold_standby.sh b/create_cold_standby.sh
new file mode 100644
index 00000000..924339af
--- /dev/null
+++ b/create_cold_standby.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+export REMOTE_SSH_KEY=/root/.ssh/id_rsa
+export REMOTE_SSH_PORT=22
+export REMOTE_SSH_HOST=my.remote.host
+
+/opt/mailcow-dockerized/helper-scripts/_cold-standby.sh
diff --git a/helper-scripts/_cold-standby.sh b/helper-scripts/_cold-standby.sh
new file mode 100755
index 00000000..3924268e
--- /dev/null
+++ b/helper-scripts/_cold-standby.sh
@@ -0,0 +1,251 @@
+#!/usr/bin/env bash
+
+PATH=$PATH:/opt/bin
+DATE=$(date +%Y-%m-%d_%H_%M_%S)
+export LC_ALL=C
+
+echo
+echo "If this script is run automatically by cron or a timer AND you are using block-level snapshots on your backup destination, make sure both do not run at the same time."
+echo "The snapshots of your backup destination should run AFTER the cold standby script finished to ensure consistent snapshots."
+echo
+
+function docker_garbage() {
+ IMGS_TO_DELETE=()
+
+ for container in $(grep -oP "image: \Kmailcow.+" docker-compose.yml); do
+
+ REPOSITORY=${container/:*}
+ TAG=${container/*:}
+ V_MAIN=${container/*.}
+ V_SUB=${container/*.}
+ EXISTING_TAGS=$(docker images | grep ${REPOSITORY} | awk '{ print $2 }')
+
+ for existing_tag in ${EXISTING_TAGS[@]}; do
+
+ V_MAIN_EXISTING=${existing_tag/*.}
+ V_SUB_EXISTING=${existing_tag/*.}
+
+ # Not an integer
+ [[ ! $V_MAIN_EXISTING =~ ^[0-9]+$ ]] && continue
+ [[ ! $V_SUB_EXISTING =~ ^[0-9]+$ ]] && continue
+
+ if [[ $V_MAIN_EXISTING == "latest" ]]; then
+ echo "Found deprecated label \"latest\" for repository $REPOSITORY, it should be deleted."
+ IMGS_TO_DELETE+=($REPOSITORY:$existing_tag)
+ elif [[ $V_MAIN_EXISTING -lt $V_MAIN ]]; then
+ echo "Found tag $existing_tag for $REPOSITORY, which is older than the current tag $TAG and should be deleted."
+ IMGS_TO_DELETE+=($REPOSITORY:$existing_tag)
+ elif [[ $V_SUB_EXISTING -lt $V_SUB ]]; then
+ echo "Found tag $existing_tag for $REPOSITORY, which is older than the current tag $TAG and should be deleted."
+ IMGS_TO_DELETE+=($REPOSITORY:$existing_tag)
+ fi
+
+ done
+
+ done
+
+ if [[ ! -z ${IMGS_TO_DELETE[*]} ]]; then
+ docker rmi ${IMGS_TO_DELETE[*]}
+ fi
+}
+
+function preflight_local_checks() {
+ if [[ -z "${REMOTE_SSH_KEY}" ]]; then
+ echo -e "\e[31mREMOTE_SSH_KEY is not set\e[0m"
+ exit 1
+ fi
+
+ if [[ ! -s "${REMOTE_SSH_KEY}" ]]; then
+ echo -e "\e[31mKeyfile ${REMOTE_SSH_KEY} is empty\e[0m"
+ exit 1
+ fi
+
+ if [[ $(stat -c "%a" "${REMOTE_SSH_KEY}") -ne 600 ]]; then
+ echo -e "\e[31mKeyfile ${REMOTE_SSH_KEY} has insecure permissions\e[0m"
+ exit 1
+ fi
+
+ if [[ ! -z "${REMOTE_SSH_PORT}" ]]; then
+ if [[ ${REMOTE_SSH_PORT} != ?(-)+([0-9]) ]] || [[ ${REMOTE_SSH_PORT} -gt 65535 ]]; then
+ echo -e "\e[31mREMOTE_SSH_PORT is set but not an integer < 65535\e[0m"
+ exit 1
+ fi
+ fi
+
+ if [[ -z "${REMOTE_SSH_HOST}" ]]; then
+ echo -e "\e[31mREMOTE_SSH_HOST cannot be empty\e[0m"
+ exit 1
+ fi
+
+ for bin in rsync docker-compose docker grep cut; do
+ if [[ -z $(which ${bin}) ]]; then
+ echo -e "\e[31mCannot find ${bin} in local PATH, exiting...\e[0m"
+ exit 1
+ fi
+ done
+
+ if grep --help 2>&1 | head -n 1 | grep -q -i "busybox"; then
+ echo -e "\e[31mBusyBox grep detected on local system, please install GNU grep\e[0m"
+ exit 1
+ fi
+}
+
+function preflight_remote_checks() {
+
+ ssh -o StrictHostKeyChecking=no \
+ -i "${REMOTE_SSH_KEY}" \
+ ${REMOTE_SSH_HOST} \
+ -p ${REMOTE_SSH_PORT} \
+ rsync -V > /dev/null
+
+ if [ $? -ne 0 ]; then
+ echo -e "\e[31mCould not verify connection to ${REMOTE_SSH_HOST}\e[0m"
+ echo -e "\e[31mPlease check the output above (is rsync >= 3.1.0 installed on the remote system?)\e[0m"
+ exit 1
+ fi
+
+ if ssh -o StrictHostKeyChecking=no \
+ -i "${REMOTE_SSH_KEY}" \
+ ${REMOTE_SSH_HOST} \
+ -p ${REMOTE_SSH_PORT} \
+ grep --help 2>&1 | head -n 1 | grep -q -i "busybox" ; then
+ echo -e "\e[31mBusyBox grep detected on remote system ${REMOTE_SSH_HOST}, please install GNU grep\e[0m"
+ exit 1
+ fi
+
+ for bin in rsync docker-compose docker; do
+ if ! ssh -o StrictHostKeyChecking=no \
+ -i "${REMOTE_SSH_KEY}" \
+ ${REMOTE_SSH_HOST} \
+ -p ${REMOTE_SSH_PORT} \
+ which ${bin} > /dev/null ; then
+ echo -e "\e[31mCannot find ${bin} in remote PATH, exiting...\e[0m"
+ exit 1
+ fi
+ done
+
+}
+
+preflight_local_checks
+preflight_remote_checks
+
+SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+COMPOSE_FILE="${SCRIPT_DIR}/../docker-compose.yml"
+source "${SCRIPT_DIR}/../mailcow.conf"
+CMPS_PRJ=$(echo $COMPOSE_PROJECT_NAME | tr -cd "[A-Za-z-_]")
+SQLIMAGE=$(grep -iEo '(mysql|mariadb)\:.+' "${COMPOSE_FILE}")
+
+echo
+echo -e "\033[1mFound compose project name ${CMPS_PRJ} for ${MAILCOW_HOSTNAME}\033[0m"
+echo -e "\033[1mFound SQL ${SQLIMAGE}\033[0m"
+echo
+
+# Make sure destination exists, rsync can fail under some circumstances
+echo -e "\033[1mPreparing remote...\033[0m"
+ssh -o StrictHostKeyChecking=no \
+ -i "${REMOTE_SSH_KEY}" \
+ ${REMOTE_SSH_HOST} \
+ -p ${REMOTE_SSH_PORT} \
+ mkdir -p "${SCRIPT_DIR}/../"
+
+# Syncing the mailcow base directory
+echo -e "\033[1mSynchronizing mailcow base directory...\033[0m"
+rsync --delete -aH -e "ssh -o StrictHostKeyChecking=no \
+ -i \"${REMOTE_SSH_KEY}\" \
+ -p ${REMOTE_SSH_PORT}" \
+ "${SCRIPT_DIR}/../" root@${REMOTE_SSH_HOST}:"${SCRIPT_DIR}/../"
+
+# Trigger a Redis save for a consistent Redis copy
+echo -ne "\033[1mRunning redis-cli save... \033[0m"
+docker exec $(docker ps -qf name=redis-mailcow) redis-cli save
+
+# Syncing volumes related to compose project
+# Same here: make sure destination exists
+for vol in $(docker volume ls -qf name="${CMPS_PRJ}"); do
+
+ mountpoint="$(docker inspect $vol | grep Mountpoint | cut -d '"' -f4)"
+
+ echo -e "\033[1mCreating remote mountpoint ${mountpoint} for ${vol}...\033[0m"
+
+ ssh -o StrictHostKeyChecking=no \
+ -i "${REMOTE_SSH_KEY}" \
+ ${REMOTE_SSH_HOST} \
+ -p ${REMOTE_SSH_PORT} \
+ mkdir -p "${mountpoint}"
+
+ if [[ "${vol}" =~ "mysql-vol-1" ]]; then
+
+ # Make sure a previous backup does not exist
+ rm -rf "${SCRIPT_DIR}/../_tmp_mariabackup/"
+
+ echo -e "\033[1mCreating consistent backup for MariaDB volume...\033[0m"
+ if ! docker run --rm \
+ --network $(docker network ls -qf name=${CMPS_PRJ}_) \
+ -v $(docker volume ls -qf name=${CMPS_PRJ}_mysql-vol-1):/var/lib/mysql/:ro \
+ --entrypoint= \
+ -v "${SCRIPT_DIR}/../_tmp_mariabackup":/backup \
+ ${SQLIMAGE} mariabackup --host mysql --user root --password ${DBROOT} --backup --target-dir=/backup 2>/dev/null ; then
+ echo -e "\e[31mFATAL: mariabackup failed, exiting\e[0m"
+ rm -rf "${SCRIPT_DIR}/../_tmp_mariabackup/"
+ exit 1
+ fi
+
+ if ! docker run --rm \
+ --network $(docker network ls -qf name=${CMPS_PRJ}_) \
+ --entrypoint= \
+ -v "${SCRIPT_DIR}/../_tmp_mariabackup":/backup \
+ ${SQLIMAGE} mariabackup --prepare --target-dir=/backup 2> /dev/null ; then
+ echo -e "\e[31mFATAL: mariabackup failed, exiting\e[0m"
+ rm -rf "${SCRIPT_DIR}/../_tmp_mariabackup/"
+ exit 1
+ fi
+
+ chown -R 999:999 "${SCRIPT_DIR}/../_tmp_mariabackup"
+
+ echo -e "\033[1mSynchronizing MariaDB backup...\033[0m"
+ rsync --delete --info=progress2 -aH -e "ssh -o StrictHostKeyChecking=no \
+ -i \"${REMOTE_SSH_KEY}\" \
+ -p ${REMOTE_SSH_PORT}" \
+ "${SCRIPT_DIR}/../_tmp_mariabackup/" root@${REMOTE_SSH_HOST}:"${mountpoint}"
+
+ # Cleanup
+ rm -rf "${SCRIPT_DIR}/../_tmp_mariabackup/"
+
+ else
+
+ echo -e "\033[1mSynchronizing ${vol} from local ${mountpoint}...\033[0m"
+ rsync --delete --info=progress2 -aH -e "ssh -o StrictHostKeyChecking=no \
+ -i \"${REMOTE_SSH_KEY}\" \
+ -p ${REMOTE_SSH_PORT}" \
+ "${mountpoint}/" root@${REMOTE_SSH_HOST}:"${mountpoint}"
+
+ fi
+
+ echo -e "\e[32mCompleted\e[0m"
+
+done
+
+# Restart Dockerd on destination
+echo -ne "\033[1mRestarting Docker daemon on remote to detect new volumes... \033[0m"
+ssh -o StrictHostKeyChecking=no \
+ -i "${REMOTE_SSH_KEY}" \
+ ${REMOTE_SSH_HOST} \
+ -p ${REMOTE_SSH_PORT} \
+ systemctl restart docker
+echo "OK"
+
+echo -e "\033[1mPulling images on remote...\033[0m"
+ssh -o StrictHostKeyChecking=no \
+ -i "${REMOTE_SSH_KEY}" \
+ ${REMOTE_SSH_HOST} \
+ -p ${REMOTE_SSH_PORT} \
+ docker-compose -f "${SCRIPT_DIR}/../docker-compose.yml" pull --no-parallel
+
+echo -e "\033[1mForcing garbage cleanup on remote...\033[0m"
+ssh -o StrictHostKeyChecking=no \
+ -i "${REMOTE_SSH_KEY}" \
+ ${REMOTE_SSH_HOST} \
+ -p ${REMOTE_SSH_PORT} \
+ ${SCRIPT_DIR}/../update.sh -f --gc
+
+echo -e "\e[32mDone\e[0m"