pub/imap-copier
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
0b12d2ed2d49b2c1640b6cf4dfd1f9d712f0946f
imap-copier/scripts/e2e.sh
T
vasyanskandClaude Opus 5 0b12d2ed2d Cover pause and resume in the e2e run 
The idempotency assertion compared counter deltas between the two runs,
but per-run counters are reset at the start of every run, so the second
run's row already showed that run alone — the delta was negative and the
script failed before reaching anything else.

With that fixed, a second account of 3000 messages exercises the new
stop path end to end: pause mid-folder, assert the task and the account
settle into paused, resume, and assert the resumed run covers every
message while skipping the ones the paused stretch had already copied.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-29 11:21:02 +07:00

286 lines
11 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# End-to-end test against the full docker-compose stack:
# postgres + app (this repo's image) + caddy + a greenmail server that plays
# BOTH the source and destination IMAP endpoint (two mailboxes on one host).
#
# Drives the real REST API: login -> create endpoints -> create task ->
# add account -> /test -> poll -> /run -> poll -> assert copied>0.
# Then runs /run a SECOND time and asserts it copies nothing new (idempotency
# via Message-ID dedup), proving the full stack end-to-end.
#
# Usage: bash scripts/e2e.sh
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT_DIR"
PROJECT="imapcopier-e2e"
ENV_FILE="$ROOT_DIR/.env.e2e"
COMPOSE=(docker compose -p "$PROJECT" --env-file "$ENV_FILE" -f docker-compose.yml -f scripts/docker-compose.e2e.yml)
HTTP_PORT=8089
BASE="http://localhost:${HTTP_PORT}"
AUTH_USER="e2e"
AUTH_PASS="e2e-$(openssl rand -hex 8)"
COOKIE_JAR="$(mktemp -t imapcopier-e2e-cookies.XXXXXX)"
SEED_PY="$(mktemp -t imapcopier-e2e-seed.XXXXXX.py)"
SRC_USER="src1@example.com"
DST_USER="dst1@example.com"
MAIL_PASS="anypass"
log() { echo "[e2e] $*"; }
fail() { echo "[e2e] FAIL: $*"; exit 1; }
cleanup() {
log "cleaning up (containers, volumes, temp files)"
"${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
rm -f "$ENV_FILE" "$COOKIE_JAR" "$SEED_PY"
}
trap cleanup EXIT
log "writing ephemeral env file $ENV_FILE"
cat > "$ENV_FILE" <<EOF
POSTGRES_PASSWORD=e2e-$(openssl rand -hex 12)
AUTH_USER=${AUTH_USER}
AUTH_PASS=${AUTH_PASS}
ENC_KEY=$(openssl rand -base64 32)
SESSION_SECRET=e2e-$(openssl rand -hex 24)
WORKER_CONCURRENCY=4
HTTP_PORT=${HTTP_PORT}
DOMAIN=
ACME_EMAIL=
EOF
log "tearing down any previous e2e stack"
"${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true
log "building and starting stack (postgres, app, caddy, greenmail)"
"${COMPOSE[@]}" up -d --build
wait_for() {
local desc="$1" tries="$2"; shift 2
for ((i = 1; i <= tries; i++)); do
if "$@" >/dev/null 2>&1; then
log "$desc: ready"
return 0
fi
sleep 1
done
fail "$desc: timed out after ${tries}s"
}
log "waiting for app healthz"
wait_for "app /healthz" 60 curl -fsS "$BASE/healthz"
log "waiting for greenmail IMAP port"
wait_for "greenmail:3143" 30 bash -c "exec 3<>/dev/tcp/127.0.0.1/3143"
log "seeding two messages into ${SRC_USER} INBOX"
cat > "$SEED_PY" <<'PYEOF'
import imaplib, sys, time
host, port, user, password, count = sys.argv[1], int(sys.argv[2]), sys.argv[3], sys.argv[4], int(sys.argv[5])
m = imaplib.IMAP4(host, port)
m.login(user, password)
m.select("INBOX")
for i in range(count):
msg = (
f"Message-ID: <e2e-{i}-{int(time.time())}@example.com>\r\n"
f"From: sender@example.com\r\n"
f"To: {user}\r\n"
f"Subject: e2e test message {i}\r\n"
f"\r\n"
f"body {i}\r\n"
).encode()
typ, data = m.append("INBOX", None, imaplib.Time2Internaldate(time.time()), msg)
if typ != "OK":
print(f"append failed: {typ} {data}", file=sys.stderr)
sys.exit(1)
m.logout()
print("seeded", count, "messages")
PYEOF
python3 "$SEED_PY" 127.0.0.1 3143 "$SRC_USER" "$MAIL_PASS" 2
api() {
local method="$1" path="$2" data="${3:-}"
if [[ -n "$data" ]]; then
curl -fsS -b "$COOKIE_JAR" -c "$COOKIE_JAR" -X "$method" "$BASE$path" \
-H 'Content-Type: application/json' -d "$data"
else
curl -fsS -b "$COOKIE_JAR" -c "$COOKIE_JAR" -X "$method" "$BASE$path"
fi
}
log "logging in as ${AUTH_USER}"
curl -fsS -c "$COOKIE_JAR" -X POST "$BASE/api/login" \
-H 'Content-Type: application/json' \
-d "{\"user\":\"${AUTH_USER}\",\"pass\":\"${AUTH_PASS}\"}" >/dev/null
log "creating src/dst endpoints (both point at greenmail:3143)"
SRC_EP_ID=$(api POST /api/endpoints '{"role_label":"src","host":"greenmail","port":3143,"tls_mode":"plain"}' | jq -r .id)
DST_EP_ID=$(api POST /api/endpoints '{"role_label":"dst","host":"greenmail","port":3143,"tls_mode":"plain"}' | jq -r .id)
[[ "$SRC_EP_ID" =~ ^[0-9]+$ ]] || fail "bad src endpoint id: $SRC_EP_ID"
[[ "$DST_EP_ID" =~ ^[0-9]+$ ]] || fail "bad dst endpoint id: $DST_EP_ID"
log "src_endpoint_id=$SRC_EP_ID dst_endpoint_id=$DST_EP_ID"
log "creating task"
TASK_ID=$(api POST /api/tasks "{\"name\":\"e2e\",\"src_endpoint_id\":${SRC_EP_ID},\"dst_endpoint_id\":${DST_EP_ID},\"folder_mapping\":{}}" | jq -r .id)
[[ "$TASK_ID" =~ ^[0-9]+$ ]] || fail "bad task id: $TASK_ID"
log "task_id=$TASK_ID"
log "adding account (src -> dst)"
ACCOUNT_ID=$(api POST "/api/tasks/${TASK_ID}/accounts" \
"{\"src_login\":\"${SRC_USER}\",\"src_pass\":\"${MAIL_PASS}\",\"dst_login\":\"${DST_USER}\",\"dst_pass\":\"${MAIL_PASS}\"}" | jq -r .id)
[[ "$ACCOUNT_ID" =~ ^[0-9]+$ ]] || fail "bad account id: $ACCOUNT_ID"
log "account_id=$ACCOUNT_ID"
log "POST /test"
api POST "/api/tasks/${TASK_ID}/test" >/dev/null
wait_test_ok() {
for ((i = 1; i <= 30; i++)); do
local res src dst
res=$(api GET "/api/tasks/${TASK_ID}")
src=$(echo "$res" | jq -r '.accounts[0].test_src_status')
dst=$(echo "$res" | jq -r '.accounts[0].test_dst_status')
if [[ "$src" == "ok" && "$dst" == "ok" ]]; then
log "connection tests: src=$src dst=$dst"
return 0
fi
if [[ "$src" == "fail" || "$dst" == "fail" ]]; then
fail "connection test failed: src=$src dst=$dst"
fi
sleep 1
done
fail "connection tests did not complete in time"
}
wait_test_ok
wait_run_done() {
# Generous: the resume scenario re-scans and copies thousands of messages.
for ((i = 1; i <= 600; i++)); do
local status
status=$(api GET "/api/tasks/${TASK_ID}" | jq -r '.task.status')
if [[ "$status" == "done" ]]; then
return 0
fi
sleep 1
done
fail "run did not finish in time (last status=$status)"
}
log "POST /run (first run)"
api POST "/api/tasks/${TASK_ID}/run" >/dev/null
wait_run_done
RES1=$(api GET "/api/tasks/${TASK_ID}")
RUN1_COPIED=$(echo "$RES1" | jq -r '.accounts[0].copied')
RUN1_SKIPPED=$(echo "$RES1" | jq -r '.accounts[0].skipped')
RUN1_ERRORS=$(echo "$RES1" | jq -r '.accounts[0].errors')
log "run 1: copied=$RUN1_COPIED skipped=$RUN1_SKIPPED errors=$RUN1_ERRORS"
[[ "$RUN1_ERRORS" == "0" ]] || fail "run 1 had errors"
[[ "$RUN1_COPIED" -gt 0 ]] || fail "run 1 copied nothing (expected >0)"
log "POST /run (second run, expect idempotency)"
api POST "/api/tasks/${TASK_ID}/run" >/dev/null
wait_run_done
# Counters are reset at the start of every run, so run 2's row shows run 2
# alone: it must copy nothing and skip what run 1 already migrated.
RES2=$(api GET "/api/tasks/${TASK_ID}")
RUN2_COPIED=$(echo "$RES2" | jq -r '.accounts[0].copied')
RUN2_SKIPPED=$(echo "$RES2" | jq -r '.accounts[0].skipped')
RUN2_ERRORS=$(echo "$RES2" | jq -r '.accounts[0].errors')
log "run 2: copied=$RUN2_COPIED skipped=$RUN2_SKIPPED errors=$RUN2_ERRORS"
[[ "$RUN2_ERRORS" == "0" ]] || fail "run 2 had errors"
[[ "$RUN2_COPIED" -eq 0 ]] || fail "run 2 copied $RUN2_COPIED new messages (expected 0, not idempotent)"
[[ "$RUN2_SKIPPED" -eq "$RUN1_COPIED" ]] ||
fail "run 2 skipped $RUN2_SKIPPED of the $RUN1_COPIED messages run 1 copied"
log "run1 copied=$RUN1_COPIED skipped=$RUN1_SKIPPED; run2 copied=$RUN2_COPIED skipped=$RUN2_SKIPPED (idempotent)"
# ---------------------------------------------------------------------------
# Pause / resume: a second account with enough messages that the run is still
# in flight when the pause lands. Pausing must leave the account resumable, and
# resuming must finish it without re-copying what the first stretch already did.
# ---------------------------------------------------------------------------
SRC_USER2="src2@example.com"
DST_USER2="dst2@example.com"
# Large enough that the copy is still in flight when the pause lands — greenmail
# on a local socket copies well over a thousand small messages per second.
SEED_COUNT=3000
log "seeding ${SEED_COUNT} messages into ${SRC_USER2} INBOX (pause/resume scenario)"
python3 "$SEED_PY" 127.0.0.1 3143 "$SRC_USER2" "$MAIL_PASS" "$SEED_COUNT"
log "adding second account (src2 -> dst2)"
ACCOUNT2_ID=$(api POST "/api/tasks/${TASK_ID}/accounts" \
"{\"src_login\":\"${SRC_USER2}\",\"src_pass\":\"${MAIL_PASS}\",\"dst_login\":\"${DST_USER2}\",\"dst_pass\":\"${MAIL_PASS}\"}" | jq -r .id)
[[ "$ACCOUNT2_ID" =~ ^[0-9]+$ ]] || fail "bad second account id: $ACCOUNT2_ID"
log "account2_id=$ACCOUNT2_ID"
log "POST /test (both accounts)"
api POST "/api/tasks/${TASK_ID}/test" >/dev/null
for ((i = 1; i <= 30; i++)); do
BOTH_OK=$(api GET "/api/tasks/${TASK_ID}" |
jq -r '[.accounts[] | select(.test_src_status=="ok" and .test_dst_status=="ok")] | length')
[[ "$BOTH_OK" == "2" ]] && break
sleep 1
done
[[ "$BOTH_OK" == "2" ]] || fail "second account did not pass connection tests (ok count=$BOTH_OK)"
# Account view for account2, by id.
acct2() { api GET "/api/tasks/${TASK_ID}" | jq -r ".accounts[] | select(.id==${ACCOUNT2_ID}) | $1"; }
log "POST /run (account2 only)"
api POST "/api/tasks/${TASK_ID}/run" "{\"account_ids\":[${ACCOUNT2_ID}]}" >/dev/null
# Per-account counters are only written to the DB when a folder completes, so
# "copied so far" is invisible here — wait for the account to go running, give
# the copy a few seconds of real work, then pause mid-folder.
log "waiting for account2 to start running"
for ((i = 1; i <= 120; i++)); do
[[ "$(acct2 .status)" == "running" ]] && break
sleep 0.5
done
[[ "$(acct2 .status)" == "running" ]] || fail "account2 never reached running"
log "letting it copy for a few seconds, then pausing mid-folder"
sleep 5
curl -fsS -b "$COOKIE_JAR" -c "$COOKIE_JAR" -X POST "$BASE/api/tasks/${TASK_ID}/pause" >/dev/null ||
fail "pause rejected — the run finished before the pause landed, seed more messages"
log "waiting for the task to settle into paused"
for ((i = 1; i <= 60; i++)); do
TASK_STATUS=$(api GET "/api/tasks/${TASK_ID}" | jq -r '.task.status')
[[ "$TASK_STATUS" == "paused" ]] && break
sleep 1
done
[[ "$TASK_STATUS" == "paused" ]] || fail "task status=$TASK_STATUS after pause (expected paused)"
ACC2_STATUS=$(acct2 .status)
[[ "$ACC2_STATUS" == "paused" ]] || fail "account2 status=$ACC2_STATUS after pause (expected paused)"
log "paused (folder-level counters at copied=$(acct2 .copied) of $SEED_COUNT)"
log "POST /resume"
api POST "/api/tasks/${TASK_ID}/resume" >/dev/null
wait_run_done
RESUMED_COPIED=$(acct2 .copied)
RESUMED_SKIPPED=$(acct2 .skipped)
RESUMED_ERRORS=$(acct2 .errors)
RESUMED_TOTAL=$((RESUMED_COPIED + RESUMED_SKIPPED))
log "after resume: copied=$RESUMED_COPIED skipped=$RESUMED_SKIPPED errors=$RESUMED_ERRORS"
[[ "$RESUMED_ERRORS" == "0" ]] || fail "resumed run had errors"
# Counters reset per run, so the resumed run alone must account for every
# message: the ones it copied now plus the ones the paused stretch already did.
[[ "$RESUMED_TOTAL" -eq "$SEED_COUNT" ]] || fail "resumed run covered $RESUMED_TOTAL of $SEED_COUNT messages"
# Non-zero skipped is the proof that the paused stretch's work survived: those
# messages are in the migration journal, so the resume did not re-copy them.
[[ "$RESUMED_SKIPPED" -gt 0 ]] ||
fail "resumed run skipped nothing — the paused stretch's progress was lost"
log "PASS: idempotent re-run; pause was resumable, resume re-copied $RESUMED_COPIED and skipped $RESUMED_SKIPPED of $SEED_COUNT"
Reference in New Issue View Git Blame Copy Permalink