diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a08fbbd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +.git +.gitignore +.superpowers +.claude +swarm-report +*.md +!README.md + +web/node_modules +web/dist + +**/*.log +.env +.env.* +!.env.example + +.DS_Store diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..9b04974 --- /dev/null +++ b/.env.example @@ -0,0 +1,11 @@ +POSTGRES_PASSWORD=change-me +AUTH_USER=admin +AUTH_PASS=change-me +# 32 bytes base64-encoded: openssl rand -base64 32 +ENC_KEY= +SESSION_SECRET=change-me-long-random +WORKER_CONCURRENCY=4 +HTTP_PORT=80 +# For HTTPS + Let's Encrypt (docker-compose.tls.yml override): set a real domain and email +DOMAIN= +ACME_EMAIL= diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f28b0e5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.env +.env.* +!.env.example + +/internal/httpapi/webdist/* +!/internal/httpapi/webdist/index.html + +.DS_Store diff --git a/Caddyfile b/Caddyfile new file mode 100644 index 0000000..f56e6ea --- /dev/null +++ b/Caddyfile @@ -0,0 +1,5 @@ +# Default profile: plain HTTP on :80, no TLS termination. +# Used by: docker compose up -d +:80 { + reverse_proxy app:8080 +} diff --git a/Caddyfile.tls b/Caddyfile.tls new file mode 100644 index 0000000..bb846ce --- /dev/null +++ b/Caddyfile.tls @@ -0,0 +1,11 @@ +# HTTPS profile: automatic Let's Encrypt certificate for $DOMAIN. +# Used by: docker compose -f docker-compose.yml -f docker-compose.tls.yml up -d +# Requires DOMAIN (and ACME_EMAIL) set in .env, and the server's :80/:443 +# reachable from the internet for the ACME HTTP-01 challenge. +{ + email {$ACME_EMAIL} +} + +{$DOMAIN} { + reverse_proxy app:8080 +} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c076d39 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# syntax=docker/dockerfile:1 + +# 1) build web (React/Vite SPA) +FROM node:22-alpine AS web +WORKDIR /web +COPY web/package.json web/package-lock.json ./ +RUN npm ci +COPY web/ ./ +RUN npm run build + +# 2) build go (embed web build via internal/httpapi/webdist) +FROM golang:1.24-alpine AS go +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +# overwrite the committed webdist stub with the real SPA build +RUN rm -rf ./internal/httpapi/webdist +COPY --from=web /web/dist ./internal/httpapi/webdist +RUN CGO_ENABLED=0 go build -trimpath -o /out/server ./cmd/server + +# 3) runtime +FROM alpine:3.20 +RUN apk add --no-cache ca-certificates +WORKDIR /app +COPY --from=go /out/server /app/server +COPY migrations /app/migrations +EXPOSE 8080 +ENTRYPOINT ["/app/server"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a17262f --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +.PHONY: build up up-tls down logs test e2e + +build: + docker compose build + +up: + docker compose up -d + +up-tls: + docker compose -f docker-compose.yml -f docker-compose.tls.yml up -d + +down: + docker compose down + +logs: + docker compose logs -f + +test: + go test ./... + +e2e: + bash scripts/e2e.sh diff --git a/README.md b/README.md new file mode 100644 index 0000000..1a6ad2e --- /dev/null +++ b/README.md @@ -0,0 +1,89 @@ +# imap-copier + +Single-binary Go server (embeds the React SPA) that copies IMAP mailboxes +between a source and a destination account. Non-destructive (copy only, +never deletes), deduplicated by Message-ID, resumable/idempotent re-runs. + +## Quick start (plain HTTP on :80) + +```bash +cp .env.example .env +# generate a real 32-byte key for ENC_KEY: +sed -i '' "s|ENC_KEY=|ENC_KEY=$(openssl rand -base64 32)|" .env +# edit .env: set POSTGRES_PASSWORD, AUTH_USER, AUTH_PASS, SESSION_SECRET + +docker compose build +docker compose up -d +curl -fsS http://localhost/healthz +``` + +The app is served through Caddy on port 80 by default — no domain or TLS +required. Login with `AUTH_USER`/`AUTH_PASS` from `.env`. + +## Enabling HTTPS (Let's Encrypt) + +Set a real, publicly resolvable domain and an ACME contact email in `.env`: + +``` +DOMAIN=copier.example.com +ACME_EMAIL=you@example.com +``` + +Then start with the TLS override instead of the plain compose file: + +```bash +docker compose -f docker-compose.yml -f docker-compose.tls.yml up -d +# or: make up-tls +``` + +This swaps Caddy's config to `Caddyfile.tls`, which requests and renews a +Let's Encrypt certificate automatically for `DOMAIN` (ports 80/443 must be +reachable from the internet for the ACME HTTP-01 challenge). Switch back to +plain HTTP with `docker compose -f docker-compose.yml up -d` (or `make up`). + +## Environment variables (`.env`) + +| Var | Required | Notes | +|---|---|---| +| `POSTGRES_PASSWORD` | yes | Postgres password, also used in `DATABASE_URL` | +| `AUTH_USER` / `AUTH_PASS` | yes | Single operator login (no user table) | +| `ENC_KEY` | yes | 32 bytes, base64: `openssl rand -base64 32` | +| `SESSION_SECRET` | yes | Signs the session cookie | +| `WORKER_CONCURRENCY` | no (default 4) | Parallel accounts copied per run | +| `HTTP_PORT` | no (default 80) | Host port Caddy binds for HTTP | +| `DOMAIN` / `ACME_EMAIL` | only for HTTPS | See above | + +## Makefile targets + +```bash +make build # docker compose build +make up # docker compose up -d (plain HTTP) +make up-tls # docker compose up -d with the Let's Encrypt override +make down # docker compose down +make logs # docker compose logs -f +make test # go test ./... +make e2e # scripts/e2e.sh (full-stack E2E against greenmail) +``` + +## E2E test + +`scripts/e2e.sh` builds and starts the full stack (postgres, app, caddy) +plus a throwaway `greenmail` IMAP server acting as both the source and +destination mailbox host, then drives the real REST API: login, create +endpoints/task/account, `/test`, `/run`, and a second `/run` to prove +idempotency (nothing is re-copied). It tears everything down afterwards. + +```bash +bash scripts/e2e.sh +# or: make e2e +``` + +## Architecture + +- `cmd/server` — entrypoint: runs DB migrations, then serves HTTP. +- `internal/httpapi` — REST API + WebSocket + embedded SPA (`webdist/`). +- `internal/orchestrator` — test/run coordination, per-account concurrency. +- `internal/imapx` — IMAP connect/list/copy primitives. +- `internal/store` — Postgres access (endpoints, tasks, accounts, runs). +- `web/` — React/Vite SPA, built into `internal/httpapi/webdist` at image + build time (see `Dockerfile`). diff --git a/docker-compose.tls.yml b/docker-compose.tls.yml new file mode 100644 index 0000000..9effd73 --- /dev/null +++ b/docker-compose.tls.yml @@ -0,0 +1,9 @@ +# Override for the HTTPS/Let's Encrypt path. +# Usage: docker compose -f docker-compose.yml -f docker-compose.tls.yml up -d +# Requires DOMAIN and ACME_EMAIL to be set in .env. +services: + caddy: + volumes: + - ./Caddyfile.tls:/etc/caddy/Caddyfile:ro + - caddy_data:/data + - caddy_config:/config diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..bcfdf36 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,47 @@ +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: imap + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: imapcopier + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U imap"] + interval: 5s + timeout: 3s + retries: 5 + + app: + build: . + environment: + DATABASE_URL: postgres://imap:${POSTGRES_PASSWORD}@postgres:5432/imapcopier?sslmode=disable + AUTH_USER: ${AUTH_USER} + AUTH_PASS: ${AUTH_PASS} + ENC_KEY: ${ENC_KEY} + SESSION_SECRET: ${SESSION_SECRET} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-4} + depends_on: + postgres: + condition: service_healthy + + caddy: + image: caddy:2-alpine + ports: + - "${HTTP_PORT:-80}:80" + - "443:443" + environment: + DOMAIN: ${DOMAIN:-} + ACME_EMAIL: ${ACME_EMAIL:-} + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - caddy_data:/data + - caddy_config:/config + depends_on: + - app + +volumes: + pgdata: + caddy_data: + caddy_config: diff --git a/scripts/docker-compose.e2e.yml b/scripts/docker-compose.e2e.yml new file mode 100644 index 0000000..f429116 --- /dev/null +++ b/scripts/docker-compose.e2e.yml @@ -0,0 +1,21 @@ +# Override used only by scripts/e2e.sh: adds a greenmail IMAP server that +# acts as BOTH the source and the destination (two mailboxes, one server), +# reachable from the app container via compose DNS as host "greenmail". +services: + greenmail: + image: greenmail/standalone:latest + environment: + GREENMAIL_OPTS: >- + -Dgreenmail.setup.test.all + -Dgreenmail.hostname=0.0.0.0 + -Dgreenmail.auth.disabled + -Dgreenmail.verbose + ports: + # exposed to the host so the seeding script (running outside docker) + # can APPEND test messages into the source mailbox + - "3143:3143" + + app: + depends_on: + greenmail: + condition: service_started diff --git a/scripts/e2e.sh b/scripts/e2e.sh new file mode 100755 index 0000000..143b1c6 --- /dev/null +++ b/scripts/e2e.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# End-to-end test against the full docker-compose stack: +# postgres + app (this repo's image) + caddy + a greenmail server that plays +# BOTH the source and destination IMAP endpoint (two mailboxes on one host). +# +# Drives the real REST API: login -> create endpoints -> create task -> +# add account -> /test -> poll -> /run -> poll -> assert copied>0. +# Then runs /run a SECOND time and asserts it copies nothing new (idempotency +# via Message-ID dedup), proving the full stack end-to-end. +# +# Usage: bash scripts/e2e.sh +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +PROJECT="imapcopier-e2e" +ENV_FILE="$ROOT_DIR/.env.e2e" +COMPOSE=(docker compose -p "$PROJECT" --env-file "$ENV_FILE" -f docker-compose.yml -f scripts/docker-compose.e2e.yml) + +HTTP_PORT=8089 +BASE="http://localhost:${HTTP_PORT}" +AUTH_USER="e2e" +AUTH_PASS="e2e-$(openssl rand -hex 8)" +COOKIE_JAR="$(mktemp -t imapcopier-e2e-cookies.XXXXXX)" +SEED_PY="$(mktemp -t imapcopier-e2e-seed.XXXXXX.py)" + +SRC_USER="src1@example.com" +DST_USER="dst1@example.com" +MAIL_PASS="anypass" + +log() { echo "[e2e] $*"; } +fail() { echo "[e2e] FAIL: $*"; exit 1; } + +cleanup() { + log "cleaning up (containers, volumes, temp files)" + "${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1 || true + rm -f "$ENV_FILE" "$COOKIE_JAR" "$SEED_PY" +} +trap cleanup EXIT + +log "writing ephemeral env file $ENV_FILE" +cat > "$ENV_FILE" </dev/null 2>&1 || true + +log "building and starting stack (postgres, app, caddy, greenmail)" +"${COMPOSE[@]}" up -d --build + +wait_for() { + local desc="$1" tries="$2"; shift 2 + for ((i = 1; i <= tries; i++)); do + if "$@" >/dev/null 2>&1; then + log "$desc: ready" + return 0 + fi + sleep 1 + done + fail "$desc: timed out after ${tries}s" +} + +log "waiting for app healthz" +wait_for "app /healthz" 60 curl -fsS "$BASE/healthz" + +log "waiting for greenmail IMAP port" +wait_for "greenmail:3143" 30 bash -c "exec 3<>/dev/tcp/127.0.0.1/3143" + +log "seeding two messages into ${SRC_USER} INBOX" +cat > "$SEED_PY" <<'PYEOF' +import imaplib, sys, time + +host, port, user, password, count = sys.argv[1], int(sys.argv[2]), sys.argv[3], sys.argv[4], int(sys.argv[5]) +m = imaplib.IMAP4(host, port) +m.login(user, password) +m.select("INBOX") +for i in range(count): + msg = ( + f"Message-ID: \r\n" + f"From: sender@example.com\r\n" + f"To: {user}\r\n" + f"Subject: e2e test message {i}\r\n" + f"\r\n" + f"body {i}\r\n" + ).encode() + typ, data = m.append("INBOX", None, imaplib.Time2Internaldate(time.time()), msg) + if typ != "OK": + print(f"append failed: {typ} {data}", file=sys.stderr) + sys.exit(1) +m.logout() +print("seeded", count, "messages") +PYEOF +python3 "$SEED_PY" 127.0.0.1 3143 "$SRC_USER" "$MAIL_PASS" 2 + +api() { + local method="$1" path="$2" data="${3:-}" + if [[ -n "$data" ]]; then + curl -fsS -b "$COOKIE_JAR" -c "$COOKIE_JAR" -X "$method" "$BASE$path" \ + -H 'Content-Type: application/json' -d "$data" + else + curl -fsS -b "$COOKIE_JAR" -c "$COOKIE_JAR" -X "$method" "$BASE$path" + fi +} + +log "logging in as ${AUTH_USER}" +curl -fsS -c "$COOKIE_JAR" -X POST "$BASE/api/login" \ + -H 'Content-Type: application/json' \ + -d "{\"user\":\"${AUTH_USER}\",\"pass\":\"${AUTH_PASS}\"}" >/dev/null + +log "creating src/dst endpoints (both point at greenmail:3143)" +SRC_EP_ID=$(api POST /api/endpoints '{"role_label":"src","host":"greenmail","port":3143,"tls_mode":"plain"}' | jq -r .id) +DST_EP_ID=$(api POST /api/endpoints '{"role_label":"dst","host":"greenmail","port":3143,"tls_mode":"plain"}' | jq -r .id) +[[ "$SRC_EP_ID" =~ ^[0-9]+$ ]] || fail "bad src endpoint id: $SRC_EP_ID" +[[ "$DST_EP_ID" =~ ^[0-9]+$ ]] || fail "bad dst endpoint id: $DST_EP_ID" +log "src_endpoint_id=$SRC_EP_ID dst_endpoint_id=$DST_EP_ID" + +log "creating task" +TASK_ID=$(api POST /api/tasks "{\"name\":\"e2e\",\"src_endpoint_id\":${SRC_EP_ID},\"dst_endpoint_id\":${DST_EP_ID},\"folder_mapping\":{}}" | jq -r .id) +[[ "$TASK_ID" =~ ^[0-9]+$ ]] || fail "bad task id: $TASK_ID" +log "task_id=$TASK_ID" + +log "adding account (src -> dst)" +ACCOUNT_ID=$(api POST "/api/tasks/${TASK_ID}/accounts" \ + "{\"src_login\":\"${SRC_USER}\",\"src_pass\":\"${MAIL_PASS}\",\"dst_login\":\"${DST_USER}\",\"dst_pass\":\"${MAIL_PASS}\"}" | jq -r .id) +[[ "$ACCOUNT_ID" =~ ^[0-9]+$ ]] || fail "bad account id: $ACCOUNT_ID" +log "account_id=$ACCOUNT_ID" + +log "POST /test" +api POST "/api/tasks/${TASK_ID}/test" >/dev/null + +wait_test_ok() { + for ((i = 1; i <= 30; i++)); do + local res src dst + res=$(api GET "/api/tasks/${TASK_ID}") + src=$(echo "$res" | jq -r '.accounts[0].test_src_status') + dst=$(echo "$res" | jq -r '.accounts[0].test_dst_status') + if [[ "$src" == "ok" && "$dst" == "ok" ]]; then + log "connection tests: src=$src dst=$dst" + return 0 + fi + if [[ "$src" == "fail" || "$dst" == "fail" ]]; then + fail "connection test failed: src=$src dst=$dst" + fi + sleep 1 + done + fail "connection tests did not complete in time" +} +wait_test_ok + +wait_run_done() { + for ((i = 1; i <= 60; i++)); do + local status + status=$(api GET "/api/tasks/${TASK_ID}" | jq -r '.task.status') + if [[ "$status" == "done" ]]; then + return 0 + fi + sleep 1 + done + fail "run did not finish in time (last status=$status)" +} + +log "POST /run (first run)" +api POST "/api/tasks/${TASK_ID}/run" >/dev/null +wait_run_done + +RES1=$(api GET "/api/tasks/${TASK_ID}") +RUN1_COPIED=$(echo "$RES1" | jq -r '.accounts[0].copied') +RUN1_SKIPPED=$(echo "$RES1" | jq -r '.accounts[0].skipped') +RUN1_ERRORS=$(echo "$RES1" | jq -r '.accounts[0].errors') +log "run 1: copied=$RUN1_COPIED skipped=$RUN1_SKIPPED errors=$RUN1_ERRORS" +[[ "$RUN1_ERRORS" == "0" ]] || fail "run 1 had errors" +[[ "$RUN1_COPIED" -gt 0 ]] || fail "run 1 copied nothing (expected >0)" + +log "POST /run (second run, expect idempotency)" +api POST "/api/tasks/${TASK_ID}/run" >/dev/null +wait_run_done + +RES2=$(api GET "/api/tasks/${TASK_ID}") +RUN2_COPIED_TOTAL=$(echo "$RES2" | jq -r '.accounts[0].copied') +RUN2_SKIPPED_TOTAL=$(echo "$RES2" | jq -r '.accounts[0].skipped') +RUN2_ERRORS=$(echo "$RES2" | jq -r '.accounts[0].errors') +RUN2_COPIED_DELTA=$((RUN2_COPIED_TOTAL - RUN1_COPIED)) +RUN2_SKIPPED_DELTA=$((RUN2_SKIPPED_TOTAL - RUN1_SKIPPED)) +log "run 2: copied_delta=$RUN2_COPIED_DELTA skipped_delta=$RUN2_SKIPPED_DELTA errors=$RUN2_ERRORS" + +[[ "$RUN2_ERRORS" == "0" ]] || fail "run 2 had errors" +[[ "$RUN2_COPIED_DELTA" -eq 0 ]] || fail "run 2 copied $RUN2_COPIED_DELTA new messages (expected 0, not idempotent)" +[[ "$RUN2_SKIPPED_DELTA" -gt 0 ]] || fail "run 2 skipped delta is $RUN2_SKIPPED_DELTA (expected >0)" + +log "PASS: run1 copied=$RUN1_COPIED skipped=$RUN1_SKIPPED; run2 copied=$RUN2_COPIED_DELTA skipped=$RUN2_SKIPPED_DELTA (idempotent)"