feat: stream the metadata scan instead of Collect
CopyFolder now streams envelope metadata via Next() in a first pass (dedup + queue new messages), then streams bodies for new ones in a second pass — no more blocking Collect of the whole folder with zero feedback, and memory stays flat (only new-message meta is held). - imapx: two-pass streaming CopyFolder + CopyDeps.OnScan(scanned,total) - orchestrator: throttled 'scan' events during the metadata pass - web: per-account 'scanning folder: X/N' line under the progress bar; scan events kept out of the log to avoid flooding Verified on greenmail: idempotency and internal-date preservation still hold. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MMHQTtnQtQqL8muAXHr9kd
This commit is contained in:
@@ -301,7 +301,7 @@ func (o *Orchestrator) runAccount(ctx context.Context, task store.Task, runID in
|
||||
var baseCopied, baseSkipped int64
|
||||
var curFolder string
|
||||
var curTotal int64
|
||||
var lastEmit time.Time
|
||||
var lastEmit, lastScanEmit time.Time
|
||||
deps := imapx.CopyDeps{
|
||||
IsMigrated: func(k string) (bool, error) { return o.store.IsMigrated(ctx, a.ID, k) },
|
||||
MarkMigrated: func(folder, k string) error { return o.store.MarkMigrated(ctx, a.ID, folder, k) },
|
||||
@@ -331,6 +331,18 @@ func (o *Orchestrator) runAccount(ctx context.Context, task store.Task, runID in
|
||||
"folder": srcFolder, "dst_folder": dstFolder, "messages": total,
|
||||
}})
|
||||
},
|
||||
// Fires while streaming metadata (dedup scan) so the UI shows movement
|
||||
// before bodies start copying. Throttled to ~4/sec, always emit the last.
|
||||
OnScan: func(scanned, total int64) {
|
||||
now := time.Now()
|
||||
if now.Sub(lastScanEmit) < 250*time.Millisecond && scanned < total {
|
||||
return
|
||||
}
|
||||
lastScanEmit = now
|
||||
o.hub.Publish(wshub.Event{Type: "scan", TaskID: task.ID, Data: map[string]any{
|
||||
"account_id": a.ID, "folder": curFolder, "scanned": scanned, "folder_total": total,
|
||||
}})
|
||||
},
|
||||
}
|
||||
for _, fp := range plan {
|
||||
if actx.Err() != nil {
|
||||
|
||||
Reference in New Issue
Block a user