Add configurable retries for locked SQLite backups

Add SQLITE_TIMEOUT, BACKUP_RETRIES, and BACKUP_RETRY_DELAY
environment variables to handle "database is locked" errors during
backup. Retry the sqlite3 .backup command with configurable timeouts
and delays, and clean up any WAL/SHM files that may interfere with
subsequent attempts. Bump version to 1.2.2.

scripts/backup.sh

@@ -27,7 +27,7 @@ hc_ping() {
     || log "WARN: healthcheck ping failed: ${url}"
 }
 
-cleanup() { rm -f "$DUMP" "$ARCHIVE"; }
+cleanup() { rm -f "$DUMP" "${DUMP}-journal" "${DUMP}-wal" "${DUMP}-shm" "$ARCHIVE"; }
 
 fail() {
   log "ERROR: $*"
@@ -43,10 +43,34 @@ fail() {
 hc_ping start
 log "backup started: $DB_FILE -> ${DEST}${BACKUP_FILE}"
 
-# 1. Consistent snapshot
-rm -f "$DUMP"
-sqlite3 "$DB_FILE" ".backup '$DUMP'" || fail "sqlite3 .backup failed"
-[ -s "$DUMP" ] || fail "dump is empty"
+# 1. Consistent snapshot.
+# A live writer (grafana, or a previous pod that has not died yet) holds the
+# write lock, so `.backup` gets "database is locked". `.timeout` makes sqlite
+# wait instead of failing instantly; the loop covers longer write bursts.
+SQLITE_TIMEOUT="${SQLITE_TIMEOUT:-60000}"
+BACKUP_RETRIES="${BACKUP_RETRIES:-5}"
+BACKUP_RETRY_DELAY="${BACKUP_RETRY_DELAY:-30}"
+
+ATTEMPT=1
+while : ; do
+  rm -f "$DUMP" "${DUMP}-journal" "${DUMP}-wal" "${DUMP}-shm"
+  ERR=$(sqlite3 -cmd ".timeout $SQLITE_TIMEOUT" "$DB_FILE" ".backup '$DUMP'" 2>&1)
+  RC=$?
+
+  if [ "$RC" -eq 0 ] && [ -s "$DUMP" ]; then
+    break
+  fi
+
+  [ -n "$ERR" ] || ERR="dump is empty"
+
+  if [ "$ATTEMPT" -ge "$BACKUP_RETRIES" ]; then
+    fail "sqlite3 .backup failed after ${ATTEMPT} attempts: $ERR"
+  fi
+
+  log "WARN: .backup attempt ${ATTEMPT}/${BACKUP_RETRIES} failed: ${ERR}; retrying in ${BACKUP_RETRY_DELAY}s"
+  sleep "$BACKUP_RETRY_DELAY"
+  ATTEMPT=$((ATTEMPT + 1))
+done
 
 # 2. Verify the snapshot before shipping it anywhere
 INTEGRITY=$(sqlite3 "$DUMP" "PRAGMA integrity_check;" 2>&1) \