public
nobgit
read
NobMail
Based on mailcow: dockerized
Languages
Repository composition by tracked source files.
PHP
49%
JavaScript
35%
HTML
9%
CSS
4%
Shell
2%
Python
1%
Lua
0%
Perl
0%
Ruby
0%
SCSS
0%
Create file
Wiki Documentation
Clone
https://nobgit.com/orgs/nobgit/nobmail.git
ssh://[email protected]:2222/orgs/nobgit/nobmail.git
Commit
[Watchdog] Add Watchdog verbose logging (#4299)
* [Watchdog] Add verbose logging * [Watchdog] More verbose debugging * [Watchdog] Enable MX check for recipients Co-authored-by: andryyy <[email protected]>
6bf70cf8
data/Dockerfiles/watchdog/Dockerfile | 2 +-
data/Dockerfiles/watchdog/watchdog.sh | 40 +++++++++++++++++++++++++----------
docker-compose.yml | 5 ++---
generate_config.sh | 3 +++
update.sh | 5 +++++
5 files changed, 40 insertions(+), 15 deletions(-)
Diff
diff --git a/data/Dockerfiles/watchdog/Dockerfile b/data/Dockerfiles/watchdog/Dockerfile
index 8378edd7..00cb8e99 100644
--- a/data/Dockerfiles/watchdog/Dockerfile
+++ b/data/Dockerfiles/watchdog/Dockerfile
@@ -36,4 +36,4 @@ RUN apk add --update \
COPY watchdog.sh /watchdog.sh
COPY check_mysql_slavestatus.sh /usr/lib/nagios/plugins/check_mysql_slavestatus.sh
-CMD /watchdog.sh 2> /dev/null
+CMD /watchdog.sh
diff --git a/data/Dockerfiles/watchdog/watchdog.sh b/data/Dockerfiles/watchdog/watchdog.sh
index cd3a2881..231d0ecd 100755
--- a/data/Dockerfiles/watchdog/watchdog.sh
+++ b/data/Dockerfiles/watchdog/watchdog.sh
@@ -6,7 +6,10 @@ trap "kill 0" EXIT
# Prepare
BACKGROUND_TASKS=()
echo "Waiting for containers to settle..."
-sleep 30
+for i in {30..1}; do
+ echo "${i}"
+ sleep 1
+done
if [[ "${USE_WATCHDOG}" =~ ^([nN][oO]|[nN])+$ ]]; then
echo -e "$(date) - USE_WATCHDOG=n, skipping watchdog..."
@@ -14,6 +17,14 @@ if [[ "${USE_WATCHDOG}" =~ ^([nN][oO]|[nN])+$ ]]; then
exec $(readlink -f "$0")
fi
+if [[ "${WATCHDOG_VERBOSE}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+ SMTP_VERBOSE="--verbose"
+ set -xv
+else
+ SMTP_VERBOSE=""
+ exec 2>/dev/null
+fi
+
# Checks pipe their corresponding container name in this pipe
if [[ ! -p /tmp/com_pipe ]]; then
mkfifo /tmp/com_pipe
@@ -114,16 +125,16 @@ function mail_error() {
IFS=',' read -r -a MAIL_RCPTS <<< "${WATCHDOG_NOTIFY_EMAIL}"
for rcpt in "${MAIL_RCPTS[@]}"; do
RCPT_DOMAIN=
- #RCPT_MX=
+ RCPT_MX=
RCPT_DOMAIN=$(echo ${rcpt} | awk -F @ {'print $NF'})
- # Latest smtp-cli looks up mx via dns
- #RCPT_MX=$(dig +short ${RCPT_DOMAIN} mx | sort -n | awk '{print $2; exit}')
- #if [[ -z ${RCPT_MX} ]]; then
- # log_msg "Cannot determine MX for ${rcpt}, skipping email notification..."
- # return 1
- #fi
+ CHECK_FOR_VALID_MX=$(dig +short ${RCPT_DOMAIN} mx)
+ if [[ -z ${CHECK_FOR_VALID_MX} ]]; then
+ log_msg "Cannot determine MX for ${rcpt}, skipping email notification..."
+ return 1
+ fi
[ -f "/tmp/${1}" ] && BODY="/tmp/${1}"
timeout 10s ./smtp-cli --missing-modules-ok \
+ "${SMTP_VERBOSE}" \
--charset=UTF-8 \
--subject="${SUBJECT}" \
--body-plain="${BODY}" \
@@ -132,8 +143,15 @@ function mail_error() {
--from="watchdog@${MAILCOW_HOSTNAME}" \
--hello-host=${MAILCOW_HOSTNAME} \
--ipv4
- #--server="${RCPT_MX}"
- log_msg "Sent notification email to ${rcpt}"
+ if [[ $? -eq 1 ]]; then # exit code 1 is fine
+ log_msg "Sent notification email to ${rcpt}"
+ else
+ if [[ "${SMTP_VERBOSE}" == "" ]]; then
+ log_msg "Error while sending notification email to ${rcpt}. You can enable verbose logging by setting 'WATCHDOG_VERBOSE=y' in mailcow.conf."
+ else
+ log_msg "Error while sending notification email to ${rcpt}."
+ fi
+ fi
done
}
@@ -154,7 +172,7 @@ get_container_ip() {
CONTAINER_ID=($(printf "%s\n" "${CONTAINER_ID[@]}" | shuf))
if [[ ! -z ${CONTAINER_ID} ]]; then
for matched_container in "${CONTAINER_ID[@]}"; do
- CONTAINER_IPS=($(curl --silent --insecure https://dockerapi/containers/${matched_container}/json | jq -r '.NetworkSettings.Networks[].IPAddress'))
+ CONTAINER_IPS=($(curl --silent --insecure https://dockerapi/containers/${matched_container}/json | jq -r '.NetworkSettings.Networks[].IPAddress'))
for ip_match in "${CONTAINER_IPS[@]}"; do
# grep will do nothing if one of these vars is empty
[[ -z ${ip_match} ]] && continue
diff --git a/docker-compose.yml b/docker-compose.yml
index 7a63f497..033a6685 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -443,9 +443,7 @@ services:
- /lib/modules:/lib/modules:ro
watchdog-mailcow:
- image: mailcow/watchdog:1.94
- # Debug
- #command: /watchdog.sh
+ image: mailcow/watchdog:1.95
dns:
- ${IPV4_NETWORK:-172.22.1}.254
tmpfs:
@@ -470,6 +468,7 @@ services:
- WATCHDOG_SUBJECT=${WATCHDOG_SUBJECT:-Watchdog ALERT}
- WATCHDOG_EXTERNAL_CHECKS=${WATCHDOG_EXTERNAL_CHECKS:-n}
- WATCHDOG_MYSQL_REPLICATION_CHECKS=${WATCHDOG_MYSQL_REPLICATION_CHECKS:-n}
+ - WATCHDOG_VERBOSE=${WATCHDOG_VERBOSE:-n}
- MAILCOW_HOSTNAME=${MAILCOW_HOSTNAME}
- COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-mailcow-dockerized}
- IPV4_NETWORK=${IPV4_NETWORK:-172.22.1}
diff --git a/generate_config.sh b/generate_config.sh
index 9aea784e..93343818 100755
--- a/generate_config.sh
+++ b/generate_config.sh
@@ -287,6 +287,9 @@ WATCHDOG_NOTIFY_BAN=n
# Will only work with unmodified mailcow setups.
WATCHDOG_EXTERNAL_CHECKS=n
+# Enable watchdog verbose logging
+WATCHDOG_VERBOSE=n
+
# Max log lines per service to keep in Redis logs
LOG_LINES=9999
diff --git a/update.sh b/update.sh
index 1bbc1c5b..cf093eab 100755
--- a/update.sh
+++ b/update.sh
@@ -513,6 +513,11 @@ for option in ${CONFIG_ARRAY[@]}; do
echo '# https://mailcow.github.io/mailcow-dockerized-docs/debug-reset-tls/' >> mailcow.conf
echo 'ACME_CONTACT=' >> mailcow.conf
fi
+elif [[ ${option} == "WATCHDOG_VERBOSE" ]]; then
+ if ! grep -q ${option} mailcow.conf; then
+ echo '# Enable watchdog verbose logging' >> mailcow.conf
+ echo 'WATCHDOG_VERBOSE=n' >> mailcow.conf
+ fi
elif ! grep -q ${option} mailcow.conf; then
echo "Adding new option \"${option}\" to mailcow.conf"
echo "${option}=n" >> mailcow.conf