feat: stream the metadata scan instead of Collect

CopyFolder now streams envelope metadata via Next() in a first pass (dedup +
queue new messages), then streams bodies for new ones in a second pass —
no more blocking Collect of the whole folder with zero feedback, and memory
stays flat (only new-message meta is held).

- imapx: two-pass streaming CopyFolder + CopyDeps.OnScan(scanned,total)
- orchestrator: throttled 'scan' events during the metadata pass
- web: per-account 'scanning folder: X/N' line under the progress bar;
  scan events kept out of the log to avoid flooding

Verified on greenmail: idempotency and internal-date preservation still hold.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMHQTtnQtQqL8muAXHr9kd
This commit is contained in:
2026-07-02 13:35:08 +07:00
parent 39285a2ee7
commit a2234077df
4 changed files with 89 additions and 22 deletions
+13 -1
View File
@@ -301,7 +301,7 @@ func (o *Orchestrator) runAccount(ctx context.Context, task store.Task, runID in
var baseCopied, baseSkipped int64
var curFolder string
var curTotal int64
var lastEmit time.Time
var lastEmit, lastScanEmit time.Time
deps := imapx.CopyDeps{
IsMigrated: func(k string) (bool, error) { return o.store.IsMigrated(ctx, a.ID, k) },
MarkMigrated: func(folder, k string) error { return o.store.MarkMigrated(ctx, a.ID, folder, k) },
@@ -331,6 +331,18 @@ func (o *Orchestrator) runAccount(ctx context.Context, task store.Task, runID in
"folder": srcFolder, "dst_folder": dstFolder, "messages": total,
}})
},
// Fires while streaming metadata (dedup scan) so the UI shows movement
// before bodies start copying. Throttled to ~4/sec, always emit the last.
OnScan: func(scanned, total int64) {
now := time.Now()
if now.Sub(lastScanEmit) < 250*time.Millisecond && scanned < total {
return
}
lastScanEmit = now
o.hub.Publish(wshub.Event{Type: "scan", TaskID: task.ID, Data: map[string]any{
"account_id": a.ID, "folder": curFolder, "scanned": scanned, "folder_total": total,
}})
},
}
for _, fp := range plan {
if actx.Err() != nil {