Reduce worker concurrency from 4 to 2
Reduce idle read timeout from 120s to 60s Add batching for IMAP metadata fetches
This commit is contained in:
+66
-31
@@ -36,6 +36,34 @@ type CopyResult struct {
|
||||
Errors int
|
||||
}
|
||||
|
||||
// metaScanBatch bounds how many messages one Pass-1 metadata FETCH covers. A
|
||||
// single unbounded FETCH 1:* over a large mailbox keeps one command open for
|
||||
// the entire scan; under parallel load the server can stop responding and,
|
||||
// since go-imap has no per-command deadline, the worker wedges forever. Short
|
||||
// windows keep each command brief so the server stays responsive and ctx is
|
||||
// checked between windows.
|
||||
const metaScanBatch = 1000
|
||||
|
||||
// metaBatches tiles 1..total into contiguous, non-overlapping windows of at
|
||||
// most batchSize, covering every sequence number exactly once.
|
||||
func metaBatches(total, batchSize uint32) []imap.SeqRange {
|
||||
if total == 0 || batchSize == 0 {
|
||||
return nil
|
||||
}
|
||||
var out []imap.SeqRange
|
||||
for start := uint32(1); start <= total; start += batchSize {
|
||||
stop := start + batchSize - 1
|
||||
if stop > total {
|
||||
stop = total
|
||||
}
|
||||
out = append(out, imap.SeqRange{Start: start, Stop: stop})
|
||||
if stop == total {
|
||||
break // guard against uint32 overflow when total is near max
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CopyFolder streams messages from srcFolder on src to dstFolder on dst.
|
||||
//
|
||||
// The source folder is opened read-only (EXAMINE) and is never mutated:
|
||||
@@ -71,45 +99,52 @@ func CopyFolder(ctx context.Context, src, dst *imapclient.Client, srcFolder, dst
|
||||
internalDate time.Time
|
||||
}
|
||||
var todo []queued
|
||||
metaSet := imap.SeqSet{imap.SeqRange{Start: 1, Stop: sel.NumMessages}}
|
||||
fc := src.Fetch(metaSet, &imap.FetchOptions{
|
||||
UID: true, Envelope: true, RFC822Size: true, Flags: true, InternalDate: true,
|
||||
})
|
||||
var scanned int64
|
||||
for {
|
||||
// Scan metadata in bounded windows instead of one FETCH 1:*, so each
|
||||
// command is short (the server stays responsive) and ctx is checked on
|
||||
// every window boundary — not just between messages of one giant command.
|
||||
for _, win := range metaBatches(sel.NumMessages, metaScanBatch) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
_ = fc.Close()
|
||||
return res, err
|
||||
}
|
||||
msg := fc.Next()
|
||||
if msg == nil {
|
||||
break
|
||||
}
|
||||
buf, err := msg.Collect()
|
||||
if err != nil {
|
||||
res.Errors++
|
||||
continue
|
||||
}
|
||||
scanned++
|
||||
key := MessageKey(buf.Envelope, buf.RFC822Size)
|
||||
already, err := deps.IsMigrated(key)
|
||||
if err != nil {
|
||||
res.Errors++
|
||||
} else if already {
|
||||
res.Skipped++
|
||||
if deps.OnProgress != nil {
|
||||
deps.OnProgress(res.Copied, res.Skipped)
|
||||
fc := src.Fetch(imap.SeqSet{win}, &imap.FetchOptions{
|
||||
UID: true, Envelope: true, RFC822Size: true, Flags: true, InternalDate: true,
|
||||
})
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
_ = fc.Close()
|
||||
return res, err
|
||||
}
|
||||
msg := fc.Next()
|
||||
if msg == nil {
|
||||
break
|
||||
}
|
||||
buf, err := msg.Collect()
|
||||
if err != nil {
|
||||
res.Errors++
|
||||
continue
|
||||
}
|
||||
scanned++
|
||||
key := MessageKey(buf.Envelope, buf.RFC822Size)
|
||||
already, err := deps.IsMigrated(key)
|
||||
if err != nil {
|
||||
res.Errors++
|
||||
} else if already {
|
||||
res.Skipped++
|
||||
if deps.OnProgress != nil {
|
||||
deps.OnProgress(res.Copied, res.Skipped)
|
||||
}
|
||||
} else {
|
||||
todo = append(todo, queued{uid: buf.UID, key: key, flags: buf.Flags, internalDate: buf.InternalDate})
|
||||
}
|
||||
if deps.OnScan != nil {
|
||||
deps.OnScan(scanned, total)
|
||||
}
|
||||
} else {
|
||||
todo = append(todo, queued{uid: buf.UID, key: key, flags: buf.Flags, internalDate: buf.InternalDate})
|
||||
}
|
||||
if deps.OnScan != nil {
|
||||
deps.OnScan(scanned, total)
|
||||
if err := fc.Close(); err != nil {
|
||||
return res, fmt.Errorf("fetch meta %q: %w", srcFolder, err)
|
||||
}
|
||||
}
|
||||
if err := fc.Close(); err != nil {
|
||||
return res, fmt.Errorf("fetch meta %q: %w", srcFolder, err)
|
||||
}
|
||||
|
||||
// Pass 2: fetch bodies for the queued (new) messages, one at a time.
|
||||
for _, q := range todo {
|
||||
|
||||
Reference in New Issue
Block a user