#!/usr/bin/env bash # End-to-end test against the full docker-compose stack: # postgres + app (this repo's image) + caddy + a greenmail server that plays # BOTH the source and destination IMAP endpoint (two mailboxes on one host). # # Drives the real REST API: login -> create endpoints -> create task -> # add account -> /test -> poll -> /run -> poll -> assert copied>0. # Then runs /run a SECOND time and asserts it copies nothing new (idempotency # via Message-ID dedup), proving the full stack end-to-end. # # Usage: bash scripts/e2e.sh set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT_DIR" PROJECT="imapcopier-e2e" ENV_FILE="$ROOT_DIR/.env.e2e" COMPOSE=(docker compose -p "$PROJECT" --env-file "$ENV_FILE" -f docker-compose.yml -f scripts/docker-compose.e2e.yml) HTTP_PORT=8089 BASE="http://localhost:${HTTP_PORT}" AUTH_USER="e2e" AUTH_PASS="e2e-$(openssl rand -hex 8)" COOKIE_JAR="$(mktemp -t imapcopier-e2e-cookies.XXXXXX)" SEED_PY="$(mktemp -t imapcopier-e2e-seed.XXXXXX.py)" SRC_USER="src1@example.com" DST_USER="dst1@example.com" MAIL_PASS="anypass" log() { echo "[e2e] $*"; } fail() { echo "[e2e] FAIL: $*"; exit 1; } cleanup() { log "cleaning up (containers, volumes, temp files)" "${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true rm -f "$ENV_FILE" "$COOKIE_JAR" "$SEED_PY" } trap cleanup EXIT log "writing ephemeral env file $ENV_FILE" cat > "$ENV_FILE" </dev/null 2>&1 || true log "building and starting stack (postgres, app, caddy, greenmail)" "${COMPOSE[@]}" up -d --build wait_for() { local desc="$1" tries="$2"; shift 2 for ((i = 1; i <= tries; i++)); do if "$@" >/dev/null 2>&1; then log "$desc: ready" return 0 fi sleep 1 done fail "$desc: timed out after ${tries}s" } log "waiting for app healthz" wait_for "app /healthz" 60 curl -fsS "$BASE/healthz" log "waiting for greenmail IMAP port" wait_for "greenmail:3143" 30 bash -c "exec 3<>/dev/tcp/127.0.0.1/3143" log "seeding two messages into ${SRC_USER} INBOX" cat > "$SEED_PY" <<'PYEOF' import imaplib, sys, time host, port, user, password, count = sys.argv[1], int(sys.argv[2]), sys.argv[3], sys.argv[4], int(sys.argv[5]) m = imaplib.IMAP4(host, port) m.login(user, password) m.select("INBOX") for i in range(count): msg = ( f"Message-ID: \r\n" f"From: sender@example.com\r\n" f"To: {user}\r\n" f"Subject: e2e test message {i}\r\n" f"\r\n" f"body {i}\r\n" ).encode() typ, data = m.append("INBOX", None, imaplib.Time2Internaldate(time.time()), msg) if typ != "OK": print(f"append failed: {typ} {data}", file=sys.stderr) sys.exit(1) m.logout() print("seeded", count, "messages") PYEOF python3 "$SEED_PY" 127.0.0.1 3143 "$SRC_USER" "$MAIL_PASS" 2 api() { local method="$1" path="$2" data="${3:-}" if [[ -n "$data" ]]; then curl -fsS -b "$COOKIE_JAR" -c "$COOKIE_JAR" -X "$method" "$BASE$path" \ -H 'Content-Type: application/json' -d "$data" else curl -fsS -b "$COOKIE_JAR" -c "$COOKIE_JAR" -X "$method" "$BASE$path" fi } log "logging in as ${AUTH_USER}" curl -fsS -c "$COOKIE_JAR" -X POST "$BASE/api/login" \ -H 'Content-Type: application/json' \ -d "{\"user\":\"${AUTH_USER}\",\"pass\":\"${AUTH_PASS}\"}" >/dev/null log "creating src/dst endpoints (both point at greenmail:3143)" SRC_EP_ID=$(api POST /api/endpoints '{"role_label":"src","host":"greenmail","port":3143,"tls_mode":"plain"}' | jq -r .id) DST_EP_ID=$(api POST /api/endpoints '{"role_label":"dst","host":"greenmail","port":3143,"tls_mode":"plain"}' | jq -r .id) [[ "$SRC_EP_ID" =~ ^[0-9]+$ ]] || fail "bad src endpoint id: $SRC_EP_ID" [[ "$DST_EP_ID" =~ ^[0-9]+$ ]] || fail "bad dst endpoint id: $DST_EP_ID" log "src_endpoint_id=$SRC_EP_ID dst_endpoint_id=$DST_EP_ID" log "creating task" TASK_ID=$(api POST /api/tasks "{\"name\":\"e2e\",\"src_endpoint_id\":${SRC_EP_ID},\"dst_endpoint_id\":${DST_EP_ID},\"folder_mapping\":{}}" | jq -r .id) [[ "$TASK_ID" =~ ^[0-9]+$ ]] || fail "bad task id: $TASK_ID" log "task_id=$TASK_ID" log "adding account (src -> dst)" ACCOUNT_ID=$(api POST "/api/tasks/${TASK_ID}/accounts" \ "{\"src_login\":\"${SRC_USER}\",\"src_pass\":\"${MAIL_PASS}\",\"dst_login\":\"${DST_USER}\",\"dst_pass\":\"${MAIL_PASS}\"}" | jq -r .id) [[ "$ACCOUNT_ID" =~ ^[0-9]+$ ]] || fail "bad account id: $ACCOUNT_ID" log "account_id=$ACCOUNT_ID" log "POST /test" api POST "/api/tasks/${TASK_ID}/test" >/dev/null wait_test_ok() { for ((i = 1; i <= 30; i++)); do local res src dst res=$(api GET "/api/tasks/${TASK_ID}") src=$(echo "$res" | jq -r '.accounts[0].test_src_status') dst=$(echo "$res" | jq -r '.accounts[0].test_dst_status') if [[ "$src" == "ok" && "$dst" == "ok" ]]; then log "connection tests: src=$src dst=$dst" return 0 fi if [[ "$src" == "fail" || "$dst" == "fail" ]]; then fail "connection test failed: src=$src dst=$dst" fi sleep 1 done fail "connection tests did not complete in time" } wait_test_ok wait_run_done() { # Generous: the resume scenario re-scans and copies thousands of messages. for ((i = 1; i <= 600; i++)); do local status status=$(api GET "/api/tasks/${TASK_ID}" | jq -r '.task.status') if [[ "$status" == "done" ]]; then return 0 fi sleep 1 done fail "run did not finish in time (last status=$status)" } log "POST /run (first run)" api POST "/api/tasks/${TASK_ID}/run" >/dev/null wait_run_done RES1=$(api GET "/api/tasks/${TASK_ID}") RUN1_COPIED=$(echo "$RES1" | jq -r '.accounts[0].copied') RUN1_SKIPPED=$(echo "$RES1" | jq -r '.accounts[0].skipped') RUN1_ERRORS=$(echo "$RES1" | jq -r '.accounts[0].errors') log "run 1: copied=$RUN1_COPIED skipped=$RUN1_SKIPPED errors=$RUN1_ERRORS" [[ "$RUN1_ERRORS" == "0" ]] || fail "run 1 had errors" [[ "$RUN1_COPIED" -gt 0 ]] || fail "run 1 copied nothing (expected >0)" log "POST /run (second run, expect idempotency)" api POST "/api/tasks/${TASK_ID}/run" >/dev/null wait_run_done # Counters are reset at the start of every run, so run 2's row shows run 2 # alone: it must copy nothing and skip what run 1 already migrated. RES2=$(api GET "/api/tasks/${TASK_ID}") RUN2_COPIED=$(echo "$RES2" | jq -r '.accounts[0].copied') RUN2_SKIPPED=$(echo "$RES2" | jq -r '.accounts[0].skipped') RUN2_ERRORS=$(echo "$RES2" | jq -r '.accounts[0].errors') log "run 2: copied=$RUN2_COPIED skipped=$RUN2_SKIPPED errors=$RUN2_ERRORS" [[ "$RUN2_ERRORS" == "0" ]] || fail "run 2 had errors" [[ "$RUN2_COPIED" -eq 0 ]] || fail "run 2 copied $RUN2_COPIED new messages (expected 0, not idempotent)" [[ "$RUN2_SKIPPED" -eq "$RUN1_COPIED" ]] || fail "run 2 skipped $RUN2_SKIPPED of the $RUN1_COPIED messages run 1 copied" log "run1 copied=$RUN1_COPIED skipped=$RUN1_SKIPPED; run2 copied=$RUN2_COPIED skipped=$RUN2_SKIPPED (idempotent)" # --------------------------------------------------------------------------- # Pause / resume: a second account with enough messages that the run is still # in flight when the pause lands. Pausing must leave the account resumable, and # resuming must finish it without re-copying what the first stretch already did. # --------------------------------------------------------------------------- SRC_USER2="src2@example.com" DST_USER2="dst2@example.com" # Large enough that the copy is still in flight when the pause lands — greenmail # on a local socket copies well over a thousand small messages per second. SEED_COUNT=3000 log "seeding ${SEED_COUNT} messages into ${SRC_USER2} INBOX (pause/resume scenario)" python3 "$SEED_PY" 127.0.0.1 3143 "$SRC_USER2" "$MAIL_PASS" "$SEED_COUNT" log "adding second account (src2 -> dst2)" ACCOUNT2_ID=$(api POST "/api/tasks/${TASK_ID}/accounts" \ "{\"src_login\":\"${SRC_USER2}\",\"src_pass\":\"${MAIL_PASS}\",\"dst_login\":\"${DST_USER2}\",\"dst_pass\":\"${MAIL_PASS}\"}" | jq -r .id) [[ "$ACCOUNT2_ID" =~ ^[0-9]+$ ]] || fail "bad second account id: $ACCOUNT2_ID" log "account2_id=$ACCOUNT2_ID" log "POST /test (both accounts)" api POST "/api/tasks/${TASK_ID}/test" >/dev/null for ((i = 1; i <= 30; i++)); do BOTH_OK=$(api GET "/api/tasks/${TASK_ID}" | jq -r '[.accounts[] | select(.test_src_status=="ok" and .test_dst_status=="ok")] | length') [[ "$BOTH_OK" == "2" ]] && break sleep 1 done [[ "$BOTH_OK" == "2" ]] || fail "second account did not pass connection tests (ok count=$BOTH_OK)" # Account view for account2, by id. acct2() { api GET "/api/tasks/${TASK_ID}" | jq -r ".accounts[] | select(.id==${ACCOUNT2_ID}) | $1"; } log "POST /run (account2 only)" api POST "/api/tasks/${TASK_ID}/run" "{\"account_ids\":[${ACCOUNT2_ID}]}" >/dev/null # Per-account counters are only written to the DB when a folder completes, so # "copied so far" is invisible here — wait for the account to go running, give # the copy a few seconds of real work, then pause mid-folder. log "waiting for account2 to start running" for ((i = 1; i <= 120; i++)); do [[ "$(acct2 .status)" == "running" ]] && break sleep 0.5 done [[ "$(acct2 .status)" == "running" ]] || fail "account2 never reached running" log "letting it copy for a few seconds, then pausing mid-folder" sleep 5 curl -fsS -b "$COOKIE_JAR" -c "$COOKIE_JAR" -X POST "$BASE/api/tasks/${TASK_ID}/pause" >/dev/null || fail "pause rejected — the run finished before the pause landed, seed more messages" log "waiting for the task to settle into paused" for ((i = 1; i <= 60; i++)); do TASK_STATUS=$(api GET "/api/tasks/${TASK_ID}" | jq -r '.task.status') [[ "$TASK_STATUS" == "paused" ]] && break sleep 1 done [[ "$TASK_STATUS" == "paused" ]] || fail "task status=$TASK_STATUS after pause (expected paused)" ACC2_STATUS=$(acct2 .status) [[ "$ACC2_STATUS" == "paused" ]] || fail "account2 status=$ACC2_STATUS after pause (expected paused)" log "paused (folder-level counters at copied=$(acct2 .copied) of $SEED_COUNT)" log "POST /resume" api POST "/api/tasks/${TASK_ID}/resume" >/dev/null wait_run_done RESUMED_COPIED=$(acct2 .copied) RESUMED_SKIPPED=$(acct2 .skipped) RESUMED_ERRORS=$(acct2 .errors) RESUMED_TOTAL=$((RESUMED_COPIED + RESUMED_SKIPPED)) log "after resume: copied=$RESUMED_COPIED skipped=$RESUMED_SKIPPED errors=$RESUMED_ERRORS" [[ "$RESUMED_ERRORS" == "0" ]] || fail "resumed run had errors" # Counters reset per run, so the resumed run alone must account for every # message: the ones it copied now plus the ones the paused stretch already did. [[ "$RESUMED_TOTAL" -eq "$SEED_COUNT" ]] || fail "resumed run covered $RESUMED_TOTAL of $SEED_COUNT messages" # Non-zero skipped is the proof that the paused stretch's work survived: those # messages are in the migration journal, so the resume did not re-copy them. [[ "$RESUMED_SKIPPED" -gt 0 ]] || fail "resumed run skipped nothing — the paused stretch's progress was lost" log "PASS: idempotent re-run; pause was resumable, resume re-copied $RESUMED_COPIED and skipped $RESUMED_SKIPPED of $SEED_COUNT"