97 lines
3.0 KiB
Bash
97 lines
3.0 KiB
Bash
#!/bin/sh
|
|
|
|
set -u
|
|
|
|
BACKUP_FILE="sqlite_$(date "+%F-%H%M%S").tar.gz"
|
|
DUMP=/tmp/db.sqlite
|
|
ARCHIVE="/tmp/${BACKUP_FILE}"
|
|
DEST="${MINIO_PATH%/}/"
|
|
|
|
log() { echo "[$(date '+%F %T')] $*"; }
|
|
|
|
# healthchecks.io ping. Self-hosted instance is supported via HEALTHCHECK_URL.
|
|
# Final url: ${HEALTHCHECK_URL}/${HEALTHCHECK_UUID}[/start|/fail]
|
|
hc_ping() {
|
|
[ -n "${HEALTHCHECK_URL:-}" ] || return 0
|
|
url="${HEALTHCHECK_URL%/}"
|
|
[ -n "${HEALTHCHECK_UUID:-}" ] && url="${url}/${HEALTHCHECK_UUID}"
|
|
[ -n "${1:-}" ] && url="${url}/$1"
|
|
curl -fsS -m 10 --retry 3 -o /dev/null "$url" \
|
|
|| log "WARN: healthcheck ping failed: ${url}"
|
|
}
|
|
|
|
cleanup() { rm -f "$DUMP" "${DUMP}-journal" "${DUMP}-wal" "${DUMP}-shm" "$ARCHIVE"; }
|
|
|
|
fail() {
|
|
log "ERROR: $*"
|
|
cleanup
|
|
hc_ping fail
|
|
exit 1
|
|
}
|
|
|
|
[ -n "${DB_FILE:-}" ] || fail "DB_FILE is not set"
|
|
[ -f "$DB_FILE" ] || fail "database not found: $DB_FILE"
|
|
[ -n "${MINIO_PATH:-}" ] || fail "MINIO_PATH is not set"
|
|
|
|
hc_ping start
|
|
log "backup started: $DB_FILE -> ${DEST}${BACKUP_FILE}"
|
|
|
|
# 1. Consistent snapshot.
|
|
# A live writer (grafana, or a previous pod that has not died yet) holds the
|
|
# write lock, so `.backup` gets "database is locked". `.timeout` makes sqlite
|
|
# wait instead of failing instantly; the loop covers longer write bursts.
|
|
SQLITE_TIMEOUT="${SQLITE_TIMEOUT:-60000}"
|
|
BACKUP_RETRIES="${BACKUP_RETRIES:-5}"
|
|
BACKUP_RETRY_DELAY="${BACKUP_RETRY_DELAY:-30}"
|
|
|
|
ATTEMPT=1
|
|
while : ; do
|
|
rm -f "$DUMP" "${DUMP}-journal" "${DUMP}-wal" "${DUMP}-shm"
|
|
ERR=$(sqlite3 -cmd ".timeout $SQLITE_TIMEOUT" "$DB_FILE" ".backup '$DUMP'" 2>&1)
|
|
RC=$?
|
|
|
|
if [ "$RC" -eq 0 ] && [ -s "$DUMP" ]; then
|
|
break
|
|
fi
|
|
|
|
[ -n "$ERR" ] || ERR="dump is empty"
|
|
|
|
if [ "$ATTEMPT" -ge "$BACKUP_RETRIES" ]; then
|
|
fail "sqlite3 .backup failed after ${ATTEMPT} attempts: $ERR"
|
|
fi
|
|
|
|
log "WARN: .backup attempt ${ATTEMPT}/${BACKUP_RETRIES} failed: ${ERR}; retrying in ${BACKUP_RETRY_DELAY}s"
|
|
sleep "$BACKUP_RETRY_DELAY"
|
|
ATTEMPT=$((ATTEMPT + 1))
|
|
done
|
|
|
|
# 2. Verify the snapshot before shipping it anywhere
|
|
INTEGRITY=$(sqlite3 "$DUMP" "PRAGMA integrity_check;" 2>&1) \
|
|
|| fail "integrity_check failed to run: $INTEGRITY"
|
|
[ "$INTEGRITY" = "ok" ] || fail "integrity_check: $INTEGRITY"
|
|
|
|
# 3. Pack
|
|
tar -C /tmp -zcf "$ARCHIVE" "$(basename "$DUMP")" || fail "tar failed"
|
|
LOCAL_SIZE=$(wc -c < "$ARCHIVE" | tr -d ' ')
|
|
[ "$LOCAL_SIZE" -gt 0 ] || fail "archive is empty"
|
|
|
|
# 4. Upload
|
|
/scripts/minio_uploader.sh copy "$ARCHIVE" "$DEST" || fail "upload failed"
|
|
|
|
# 5. Verify the remote object exists and matches the local size
|
|
REMOTE_SIZE=$(/scripts/minio_uploader.sh size "${DEST}${BACKUP_FILE}")
|
|
[ -n "$REMOTE_SIZE" ] || fail "uploaded file not found at ${DEST}${BACKUP_FILE}"
|
|
[ "$REMOTE_SIZE" = "$LOCAL_SIZE" ] \
|
|
|| fail "size mismatch: local=${LOCAL_SIZE} remote=${REMOTE_SIZE}"
|
|
|
|
log "uploaded and verified: ${DEST}${BACKUP_FILE} (${LOCAL_SIZE} bytes)"
|
|
cleanup
|
|
|
|
# 6. Rotate old backups
|
|
if [ -n "${DELETE_AFTER:-}" ] && [ "${DELETE_AFTER}" -gt 0 ] 2>/dev/null; then
|
|
/scripts/deleteold.sh || fail "rotation of old backups failed"
|
|
fi
|
|
|
|
log "backup finished successfully"
|
|
hc_ping
|