Remove break-lock, add checking, and fix healthcheck

src/backup-wrapper.sh

@@ -2,15 +2,56 @@
 
 echo "Starting backup wrapper script at $(date)"
 
+mkdir -p /health || exit 1
+
+# Called indirectly by the EXIT trap, including configuration failures.
+# shellcheck disable=SC2317
+finish() {
+    local status=$?
+    if (( status != 0 )); then
+        touch /health/backup_failed || echo "Cannot record backup failure" >&2
+    fi
+    rm -f /health/backup_completion_time.log.$$
+    exit "$status"
+}
+trap 'finish' EXIT
+
+if [ -e /health/check_failed ]; then
+    echo "A previous repository check failed. Investigate and run full checks before clearing /health/check_failed." >&2
+    exit 1
+fi
+
+# shellcheck source=src/borg-common.sh
+source /src/borg-common.sh || exit 1
+
 execute_script() {
-    echo "Executing script with:"
+    echo "Executing $operation with:"
     if [ -n "$BORG_PASSPHRASE" ]; then
         echo "BORG_PASSPHRASE=<redacted>"
     fi
     echo "BORG_REMOTE_PATH='${BORG_REMOTE_PATH}'"
     echo "BORG_REPO='${BORG_REPO}'"
 
-    /src/backup.sh
+    local status=0
+    if [[ $operation == backup ]]; then
+        /src/backup.sh || status=$?
+    else
+        echo "Checking repository for up to $check_duration seconds"
+        borg_wait check --repository-only --max-duration="$check_duration" --info || status=$?
+        if (( status != 0 )); then
+            # Later partial checks may skip the damaged segment. Only clear
+            # this marker manually after investigating and fully checking it.
+            touch /health/check_failed || status=1
+            echo "Repository check failed (Borg exit status $status). Stopping backup cycle." >&2
+            exit "$status"
+        fi
+    fi
+    if (( status != 0 )); then
+        # Report failure while later targets are still running, and keep it
+        # across retries and container restarts until every target succeeds.
+        touch /health/backup_failed || return 1
+    fi
+    return "$status"
 }
 
 configure_environment() {
@@ -25,55 +66,103 @@ configure_environment() {
     fi
 }
 
+duration_seconds() {
+    local duration=$1 whole fraction multiplier seconds
+    local index product carry=0 remainder=0
+    local range_error='SLEEP_TIME must represent between 1 and 2147483647 seconds.'
+
+    if [[ ! $duration =~ ^([0-9]+([.][0-9]*)?|[.][0-9]+)([smhd]?)$ ]]; then
+        echo 'SLEEP_TIME must be a positive duration in seconds, minutes, hours or days.' >&2
+        return 1
+    fi
+    duration=${BASH_REMATCH[1]}
+    case ${BASH_REMATCH[3]} in
+        ''|s) multiplier=1 ;;
+        m) multiplier=60 ;;
+        h) multiplier=3600 ;;
+        d) multiplier=86400 ;;
+    esac
+
+    whole=${duration%%.*}
+    fraction=${duration#"$whole"}
+    fraction=${fraction#.}
+    # Strip leading zeros and bound the length before using Bash arithmetic.
+    whole=${whole#"${whole%%[!0]*}"}
+    if (( ${#whole} > 10 )); then
+        echo "$range_error" >&2
+        return 1
+    fi
+    seconds=$((10#${whole:-0} * multiplier))
+
+    # Multiply fractional digits from right to left, retaining any remainder
+    # so values above the upper limit are rejected before truncating seconds.
+    for ((index = ${#fraction} - 1; index >= 0; index--)); do
+        product=$((10#${fraction:index:1} * multiplier + carry))
+        carry=$((product / 10))
+        remainder=$((remainder || product % 10))
+    done
+    seconds=$((seconds + carry))
+    if (( seconds < 1 || seconds > 2147483647 || (seconds == 2147483647 && remainder) )); then
+        echo "$range_error" >&2
+        return 1
+    fi
+    echo "$seconds"
+}
+
 main() {
-    if [ -n "$BORG_REPO" ]; then
-        # fallback case if multi-target backup isn't needed
-        if ! execute_script; then
-            echo "Skipping completion log due to backup failure"
+    local check_budget check_duration operation
+    local index target_count=1 any_failed=false completed_indices indexed_var_name
+    # Preserve the existing SLEEP_TIME setting, including fractional durations.
+    check_budget=$(duration_seconds "${SLEEP_TIME:-1h}") || return 1
+
+    # Finish every backup before spending the former sleep interval checking.
+    for operation in backup check; do
+        check_duration=$((check_budget / target_count))
+        if (( check_duration == 0 )); then
+            echo "SLEEP_TIME must allow at least one second per repository."
             return 1
         fi
-    else
-        local index=0
-        local any_failed=false
-        local completed_indices=" "
-
-        while configure_environment "$index"; do
+        if [ -n "$BORG_REPO" ]; then
             execute_script || any_failed=true
-            completed_indices+="$index "
-            unset BORG_PASSPHRASE BORG_REMOTE_PATH BORG_REPO
-            ((index++))
-        done
+        else
+            index=0
+            completed_indices=" "
+            while configure_environment "$index"; do
+                execute_script || any_failed=true
+                completed_indices+="$index "
+                unset BORG_PASSPHRASE BORG_REMOTE_PATH BORG_REPO
+                ((index++))
+            done
+            target_count=$index
 
-        echo "Finished backup script at $(date)"
-        if (( index == 0 )); then
-            echo "No valid configuration found. Please ensure environment variables are set properly."
-            return 1
+            if (( index == 0 )); then
+                echo "No valid configuration found. Please ensure environment variables are set properly."
+                return 1
+            fi
+
+            # A missing or incomplete target must not silently hide later targets.
+            for indexed_var_name in ${!BORG_REPO_@} ${!BORG_PASSPHRASE_@} ${!BORG_REMOTE_PATH_@}; do
+                [[ -n ${!indexed_var_name} ]] || continue
+                case "$completed_indices" in
+                    *" ${indexed_var_name##*_} "*) ;;
+                    *)
+                        echo "Invalid or non-contiguous backup configuration: $indexed_var_name. Skipping completion log."
+                        return 1
+                        ;;
+                esac
+            done
         fi
+    done
 
-        # A missing or incomplete target must not silently hide later targets.
-        local indexed_var_name
-        for indexed_var_name in ${!BORG_REPO_@} ${!BORG_PASSPHRASE_@} ${!BORG_REMOTE_PATH_@}; do
-            [[ -n ${!indexed_var_name} ]] || continue
-            case "$completed_indices" in
-                *" ${indexed_var_name##*_} "*) ;;
-                *)
-                    echo "Invalid or non-contiguous backup configuration: $indexed_var_name. Skipping completion log."
-                    return 1
-                    ;;
-            esac
-        done
-
-        if [[ $any_failed == true ]]; then
-            echo "Skipping completion log due to backup failure(s)"
-            return 1
-        fi
+    if [[ $any_failed == true ]]; then
+        echo "Skipping completion log due to backup or check failure(s)"
+        return 1
     fi
 
-    # Rename only a complete timestamp, leaving the previous success intact on failure.
-    mkdir -p /health || return 1
-    trap 'rm -f /health/backup_completion_time.log.$$' EXIT
+    # Clear the failure only after publishing a complete, successful timestamp.
     date -u '+%Y-%m-%dT%H:%M:%SZ' > /health/backup_completion_time.log.$$ &&
-        mv -fT /health/backup_completion_time.log.$$ /health/backup_completion_time.log
+        mv -fT /health/backup_completion_time.log.$$ /health/backup_completion_time.log &&
+        rm -f /health/backup_failed
 }
 
 main

src/backup.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 : "${ARCHIVE_PREFIX:?Set ARCHIVE_PREFIX to a non-empty archive prefix (e.g. my-host-)}"
 
 KEEP_DAILY=${KEEP_DAILY:-6}
@@ -9,20 +11,30 @@ KEEP_YEARLY=${KEEP_YEARLY:-10}
 
 echo "Starting backup script at $(date)"
 
-export BORG_RSH='ssh -oBatchMode=yes' # https://borgbackup.readthedocs.io/en/stable/usage/notes.html#ssh-batch-mode
+# shellcheck source=src/borg-common.sh
+source /src/borg-common.sh
 
-# break any stale locks in case the script was interrupted
-borg break-lock
-
-if ! borg info; then
-    echo "Borg info returned a non-zero status. Initializing Borg..."
-    borg init --encryption=repokey
-fi
-
-# The above command will fail if the repo hasn't been already initialized,
-# so we can ignore the return status. However, if any of the commands below fail,
-# we want to stop the script immediately.
-set -e
+# Only a missing repository (modern exit code 13) triggers initialization.
+info_status=0
+borg_wait info || info_status=$?
+case "$info_status" in
+    0) ;;
+    13)
+        echo "Repository does not exist. Initializing Borg..."
+        init_status=0
+        borg_wait init --encryption=repokey || init_status=$?
+        case "$init_status" in
+            0) ;;
+            # Another client may have initialized the repository in between.
+            10) borg_wait info ;;
+            *) exit "$init_status" ;;
+        esac
+        ;;
+    *)
+        echo "Cannot access repository (Borg exit status $info_status). Skipping backup." >&2
+        exit "$info_status"
+        ;;
+esac
 
 cleanup() {
     if [ -n "${GIT_EXCLUDE_FILE:-}" ] && [ -f "$GIT_EXCLUDE_FILE" ]; then
@@ -68,7 +80,7 @@ if [ "${IGNORE_GIT_UNTRACKED:-false}" = "true" ]; then
     EXCLUDE_ARGS+=(--exclude-from "$GIT_EXCLUDE_FILE")
 fi
 
-borg create --stats \
+borg_wait create --stats \
     --list \
     --filter=AMCE \
     --files-cache=ctime,size,inode \
@@ -77,11 +89,11 @@ borg create --stats \
 
 cd -
 
-borg prune --list --stats \
+borg_wait prune --list --stats \
     --glob-archives="${ARCHIVE_PREFIX}*" \
     --keep-daily="$KEEP_DAILY" \
     --keep-weekly="$KEEP_WEEKLY" \
     --keep-monthly="$KEEP_MONTHLY" \
     --keep-yearly="$KEEP_YEARLY"
 
-borg compact --threshold=5 --cleanup-commits --verbose --progress
+borg_wait compact --threshold=5 --cleanup-commits --verbose --progress

src/borg-common.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+export BORG_RSH='ssh -oBatchMode=yes'
+
+# Never break another client's repository/cache lock. Borg 1.4 only accepts
+# finite waits, so retry lock timeouts forever; other errors must propagate.
+borg_wait() {
+    local status
+    while true; do
+        if BORG_EXIT_CODES=modern borg "$@" --lock-wait=60; then
+            return 0
+        else
+            status=$?
+        fi
+        if (( status != 73 )); then
+            return "$status"
+        fi
+        echo "Repository or cache is locked; continuing to wait..." >&2
+    done
+}

src/healthcheck.sh

@@ -13,6 +13,9 @@ if ! [ "$MAX_BACKUP_AGE_SECONDS" -gt 0 ] 2>/dev/null; then
     unhealthy "MAX_BACKUP_AGE_SECONDS must be a positive integer."
 fi
 
+[ ! -e /health/check_failed ] || unhealthy "A repository check failed. Investigate and run full checks before clearing /health/check_failed."
+[ ! -e /health/backup_failed ] || unhealthy "A backup failed. Check the backup logs; all targets must succeed to recover."
+
 timestamp_file=/health/backup_completion_time.log
 [ -f "$timestamp_file" ] || timestamp_file=/health/container_start_time.log
 

src/schedule.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
-SLEEP_TIME=${SLEEP_TIME:-1h}
+# Preserve the wrapper's failure through the logging pipeline and stop the loop.
+set -eo pipefail
 
 log_message() {
     stdbuf -o0 tee -a "$(get_log_file_name)"
@@ -17,9 +18,7 @@ date -u '+%Y-%m-%dT%H:%M:%SZ' > /health/container_start_time.log.$$ &&
     mv -fT /health/container_start_time.log.$$ /health/container_start_time.log || exit 1
 
 while true; do
+    # The wrapper checks every repository after backups, using SLEEP_TIME as
+    # its total check budget. Start the next cycle as soon as checks finish.
     /src/backup-wrapper.sh 2>&1 | log_message
-    echo "Sleeping for $SLEEP_TIME" | log_message
-
-    # Using a simple sleep loop to schedule backups instead of cron to avoid concurrency issues
-    sleep "$SLEEP_TIME"
 done